






'''simple docstring'''





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


A        =			['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']


class 							__a							(  __A						):





		'''simple docstring'''



		def __init__(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__=None ,    UpperCamelCase__=1       ):
						SCREAMING_SNAKE_CASE_   :	int													=   tokenizer
						SCREAMING_SNAKE_CASE_   :	str													=   dataset
						SCREAMING_SNAKE_CASE_   :	int													=   len(UpperCamelCase__       ) if n_tasks is None else n_tasks
						SCREAMING_SNAKE_CASE_   :	Dict													=   n_copies






		def __iter__(						self       ):
						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   []
						for task in range(self.n_tasks       ):
										# without strip, the model generate commented codes ...
										prompts.append(self.tokenizer.eos_token + self.dataset[task]['prompt'].strip()       )
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   self.tokenizer(UpperCamelCase__ ,    padding=UpperCamelCase__ ,    return_tensors='pt'       )
						for task in range(self.n_tasks       ):
										for _ in range(self.n_copies       ):
														yield {
														    "ids": outputs.input_ids[task],
														    "task_id": task,
														    "input_len": outputs.attention_mask[task].sum(),
														}






class 							__a							(  __A						):





		'''simple docstring'''



		def __init__(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Any													=   start_length
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   eof_strings
						SCREAMING_SNAKE_CASE_   :	Tuple													=   tokenizer






		def __call__(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    **UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	str													=   self.tokenizer.batch_decode(input_ids[:, self.start_length :]       )
						SCREAMING_SNAKE_CASE_   :	Tuple													=   []
						for decoded_generation in decoded_generations:
										done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings       )       )
						return all(UpperCamelCase__       )






def 					_lowerCamelCase(			lowerCAmelCase__				:		Any      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Dict													=   re.split('(%s)' % '|'.join(lowerCAmelCase__      )    ,    lowerCAmelCase__      )
				# last string should be ""
				return "".join(string_list[:-2]      )



def 					_lowerCamelCase(			lowerCAmelCase__				:		str    ,    lowerCAmelCase__				:		int    ,    lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		Optional[Any]    ,    lowerCAmelCase__				:		Optional[int]    ,    lowerCAmelCase__				:		str=20    ,    **lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   defaultdict(lowerCAmelCase__      )  # dict of list of generated tokens
				for step, batch in tqdm(enumerate(lowerCAmelCase__      )      ):
								with torch.no_grad():
												SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   batch['ids'].shape[-1]
												SCREAMING_SNAKE_CASE_   :	List[str]													=   accelerator.unwrap_model(lowerCAmelCase__      ).generate(
												    input_ids=batch['ids'][:, : batch['input_len']]    ,    num_return_sequences=lowerCAmelCase__    ,    **lowerCAmelCase__      )
												# each task is generated batch_size times
												SCREAMING_SNAKE_CASE_   :	List[Any]													=   batch['task_id'].repeat(lowerCAmelCase__      )
												SCREAMING_SNAKE_CASE_   :	Optional[int]													=   accelerator.pad_across_processes(
												    lowerCAmelCase__    ,    dim=1    ,    pad_index=tokenizer.pad_token_id      )

												SCREAMING_SNAKE_CASE_   :	Tuple													=   accelerator.gather((generated_tokens, generated_tasks)      )
												SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   generated_tokens.cpu().numpy()
												SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   generated_tasks.cpu().numpy()

												for task, generated_tokens in zip(lowerCAmelCase__    ,    lowerCAmelCase__      ):
																gen_token_dict[task].append(lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	Dict													=   [[] for _ in range(lowerCAmelCase__      )]
				for task, generated_tokens in gen_token_dict.items():
								for s in generated_tokens:
												SCREAMING_SNAKE_CASE_   :	Any													=   tokenizer.decode(lowerCAmelCase__    ,    skip_special_tokens=lowerCAmelCase__    ,    clean_up_tokenization_spaces=lowerCAmelCase__      )
												code_gens[task].append(remove_last_block(lowerCAmelCase__      )      )
				return code_gens



def 					_lowerCamelCase(			):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Tuple													=   HfArgumentParser(lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	List[str]													=   parser.parse_args()

				transformers.logging.set_verbosity_error()
				# enables code execution in code_eval metric
				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   args.HF_ALLOW_CODE_EVAL
				# make sure tokenizer plays nice with multiprocessing
				SCREAMING_SNAKE_CASE_   :	Dict													=   'false'

				if args.num_workers is None:
								SCREAMING_SNAKE_CASE_   :	int													=   multiprocessing.cpu_count()

				# Use dataset load to feed to accelerate
				SCREAMING_SNAKE_CASE_   :	Dict													=   Accelerator()
				set_seed(args.seed    ,    device_specific=lowerCAmelCase__      )

				# Load model and tokenizer
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   AutoTokenizer.from_pretrained(args.model_ckpt      )
				SCREAMING_SNAKE_CASE_   :	Dict													=   tokenizer.eos_token
				SCREAMING_SNAKE_CASE_   :	int													=   AutoModelForCausalLM.from_pretrained(args.model_ckpt      )

				# Generation settings
				SCREAMING_SNAKE_CASE_   :	int													=   {
				    'do_sample': args.do_sample,
				    'temperature': args.temperature,
				    'max_new_tokens': args.max_new_tokens,
				    'top_p': args.top_p,
				    'top_k': args.top_k,
				    'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0    ,    lowerCAmelCase__    ,    lowerCAmelCase__      )]      ),
				}

				# Load evaluation dataset and metric
				SCREAMING_SNAKE_CASE_   :	Tuple													=   load_dataset('openai_humaneval'      )
				SCREAMING_SNAKE_CASE_   :	Tuple													=   load_metric('code_eval'      )

				SCREAMING_SNAKE_CASE_   :	Dict													=   args.num_tasks if args.num_tasks is not None else len(human_eval['test']      )
				SCREAMING_SNAKE_CASE_   :	List[str]													=   args.n_samples // args.batch_size

				SCREAMING_SNAKE_CASE_   :	int													=   TokenizedDataset(lowerCAmelCase__    ,    human_eval['test']    ,    n_copies=lowerCAmelCase__    ,    n_tasks=lowerCAmelCase__      )
				# do not confuse args.batch_size, which is actually the num_return_sequences
				SCREAMING_SNAKE_CASE_   :	Any													=   DataLoader(lowerCAmelCase__    ,    batch_size=1      )

				# Run a quick test to see if code evaluation is enabled
				try:
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   code_eval_metric.compute(references=['']    ,    predictions=[['']]      )
				except ValueError as exception:
								print(
								    'Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'
								    ' flag to enable code evaluation.'      )
								raise exception

				SCREAMING_SNAKE_CASE_   :	List[str]													=   accelerator.prepare(lowerCAmelCase__    ,    lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	int													=   complete_code(
				    lowerCAmelCase__    ,    lowerCAmelCase__    ,    lowerCAmelCase__    ,    lowerCAmelCase__    ,    n_tasks=lowerCAmelCase__    ,    batch_size=args.batch_size    ,    **lowerCAmelCase__    ,    )

				if accelerator.is_main_process:
								SCREAMING_SNAKE_CASE_   :	str													=   []

								for task in tqdm(range(lowerCAmelCase__      )      ):
												SCREAMING_SNAKE_CASE_   :	int													=   human_eval['test'][task]['test']
												SCREAMING_SNAKE_CASE_   :	Optional[int]													=   F'''check({human_eval['test'][task]['entry_point']})'''
												references.append('\n' + test_func + '\n' + entry_point      )

								# Evaluate completions with "code_eval" metric
								SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   code_eval_metric.compute(
								    references=lowerCAmelCase__    ,    predictions=lowerCAmelCase__    ,    num_workers=args.num_workers      )
								print(F'''Results: {pass_at_k}'''      )

								# Save results to json file
								with open(args.output_file    ,    'w'      ) as fp:
												json.dump(lowerCAmelCase__    ,    lowerCAmelCase__      )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
				main()







import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
				import torch

if is_vision_available():
				import PIL


A        =			logging.get_logger(__name__)



def 					_lowerCamelCase(			lowerCAmelCase__				:		np.ndarray    ,    lowerCAmelCase__				:		Union[int, Iterable[int]]    ,    lowerCAmelCase__				:		bool    ,    lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				def constraint_to_multiple_of(lowerCAmelCase__				:		str    ,    lowerCAmelCase__				:		Optional[Any]    ,    lowerCAmelCase__				:		Optional[Any]=0    ,    lowerCAmelCase__				:		Any=None      ):
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   round(val / multiple      ) * multiple

								if max_val is not None and x > max_val:
												SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   math.floor(val / multiple      ) * multiple

								if x < min_val:
												SCREAMING_SNAKE_CASE_   :	Dict													=   math.ceil(val / multiple      ) * multiple

								return x

				SCREAMING_SNAKE_CASE_   :	str													=   (output_size, output_size) if isinstance(lowerCAmelCase__    ,    lowerCAmelCase__      ) else output_size

				SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Any													=   get_image_size(lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	List[str]													=   output_size

				# determine new height and width
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   output_height / input_height
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   output_width / input_width

				if keep_aspect_ratio:
								# scale as little as possible
								if abs(1 - scale_width      ) < abs(1 - scale_height      ):
												# fit width
												SCREAMING_SNAKE_CASE_   :	str													=   scale_width
								else:
												# fit height
												SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   scale_height

				SCREAMING_SNAKE_CASE_   :	Any													=   constraint_to_multiple_of(scale_height * input_height    ,    multiple=lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	int													=   constraint_to_multiple_of(scale_width * input_width    ,    multiple=lowerCAmelCase__      )

				return (new_height, new_width)


class 							__a							(  __A						):





		'''simple docstring'''



		UpperCAmelCase__      :				List[Any]				    =						["""pixel_values"""]


		def __init__(						self ,    UpperCamelCase__ = True ,    UpperCamelCase__ = None ,    UpperCamelCase__ = PILImageResampling.BILINEAR ,    UpperCamelCase__ = False ,    UpperCamelCase__ = 1 ,    UpperCamelCase__ = True ,    UpperCamelCase__ = 1 / 255 ,    UpperCamelCase__ = True ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    **UpperCamelCase__ ,    ):
						super().__init__(**UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_   :	Any													=   size if size is not None else {'height': 384, 'width': 384}
						SCREAMING_SNAKE_CASE_   :	str													=   get_size_dict(UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_   :	str													=   do_resize
						SCREAMING_SNAKE_CASE_   :	str													=   size
						SCREAMING_SNAKE_CASE_   :	int													=   keep_aspect_ratio
						SCREAMING_SNAKE_CASE_   :	Tuple													=   ensure_multiple_of
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   resample
						SCREAMING_SNAKE_CASE_   :	int													=   do_rescale
						SCREAMING_SNAKE_CASE_   :	int													=   rescale_factor
						SCREAMING_SNAKE_CASE_   :	str													=   do_normalize
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						SCREAMING_SNAKE_CASE_   :	int													=   image_std if image_std is not None else IMAGENET_STANDARD_STD


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__ = False ,    UpperCamelCase__ = 1 ,    UpperCamelCase__ = PILImageResampling.BICUBIC ,    UpperCamelCase__ = None ,    **UpperCamelCase__ ,    ):
						SCREAMING_SNAKE_CASE_   :	List[Any]													=   get_size_dict(UpperCamelCase__       )
						if "height" not in size or "width" not in size:
										raise ValueError(F'''The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'''       )
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   get_resize_output_image_size(
						    UpperCamelCase__ ,    output_size=(size['height'], size['width']) ,    keep_aspect_ratio=UpperCamelCase__ ,    multiple=UpperCamelCase__ ,    )
						return resize(UpperCamelCase__ ,    size=UpperCamelCase__ ,    resample=UpperCamelCase__ ,    data_format=UpperCamelCase__ ,    **UpperCamelCase__       )


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__ = None ,    **UpperCamelCase__ ,    ):
						return rescale(UpperCamelCase__ ,    scale=UpperCamelCase__ ,    data_format=UpperCamelCase__ ,    **UpperCamelCase__       )


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__ = None ,    **UpperCamelCase__ ,    ):
						return normalize(UpperCamelCase__ ,    mean=UpperCamelCase__ ,    std=UpperCamelCase__ ,    data_format=UpperCamelCase__ ,    **UpperCamelCase__       )


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = None ,    UpperCamelCase__ = ChannelDimension.FIRST ,    **UpperCamelCase__ ,    ):
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   do_resize if do_resize is not None else self.do_resize
						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   size if size is not None else self.size
						SCREAMING_SNAKE_CASE_   :	str													=   get_size_dict(UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_   :	Dict													=   keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
						SCREAMING_SNAKE_CASE_   :	str													=   ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   resample if resample is not None else self.resample
						SCREAMING_SNAKE_CASE_   :	Dict													=   do_rescale if do_rescale is not None else self.do_rescale
						SCREAMING_SNAKE_CASE_   :	Dict													=   rescale_factor if rescale_factor is not None else self.rescale_factor
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   do_normalize if do_normalize is not None else self.do_normalize
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   image_mean if image_mean is not None else self.image_mean
						SCREAMING_SNAKE_CASE_   :	Dict													=   image_std if image_std is not None else self.image_std

						SCREAMING_SNAKE_CASE_   :	List[Any]													=   make_list_of_images(UpperCamelCase__       )

						if not valid_images(UpperCamelCase__       ):
										raise ValueError(
										    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
										    'torch.Tensor, tf.Tensor or jax.ndarray.'       )

						if do_resize and size is None or resample is None:
										raise ValueError('Size and resample must be specified if do_resize is True.'       )

						if do_rescale and rescale_factor is None:
										raise ValueError('Rescale factor must be specified if do_rescale is True.'       )

						if do_normalize and (image_mean is None or image_std is None):
										raise ValueError('Image mean and std must be specified if do_normalize is True.'       )

						# All transformations expect numpy arrays.
						SCREAMING_SNAKE_CASE_   :	Any													=   [to_numpy_array(UpperCamelCase__       ) for image in images]

						if do_resize:
										SCREAMING_SNAKE_CASE_   :	Tuple													=   [self.resize(image=UpperCamelCase__ ,    size=UpperCamelCase__ ,    resample=UpperCamelCase__       ) for image in images]

						if do_rescale:
										SCREAMING_SNAKE_CASE_   :	Any													=   [self.rescale(image=UpperCamelCase__ ,    scale=UpperCamelCase__       ) for image in images]

						if do_normalize:
										SCREAMING_SNAKE_CASE_   :	Any													=   [self.normalize(image=UpperCamelCase__ ,    mean=UpperCamelCase__ ,    std=UpperCamelCase__       ) for image in images]

						SCREAMING_SNAKE_CASE_   :	str													=   [to_channel_dimension_format(UpperCamelCase__ ,    UpperCamelCase__       ) for image in images]

						SCREAMING_SNAKE_CASE_   :	Dict													=   {'pixel_values': images}
						return BatchFeature(data=UpperCamelCase__ ,    tensor_type=UpperCamelCase__       )






		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None       ):
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   outputs.logits

						# Resize logits and compute semantic segmentation maps
						if target_sizes is not None:
										if len(UpperCamelCase__       ) != len(UpperCamelCase__       ):
														raise ValueError(
														    'Make sure that you pass in as many target sizes as the batch dimension of the logits'       )

										if is_torch_tensor(UpperCamelCase__       ):
														SCREAMING_SNAKE_CASE_   :	int													=   target_sizes.numpy()

										SCREAMING_SNAKE_CASE_   :	Tuple													=   []

										for idx in range(len(UpperCamelCase__       )       ):
														SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   torch.nn.functional.interpolate(
														    logits[idx].unsqueeze(dim=0       ) ,    size=target_sizes[idx] ,    mode='bilinear' ,    align_corners=UpperCamelCase__       )
														SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   resized_logits[0].argmax(dim=0       )
														semantic_segmentation.append(UpperCamelCase__       )
						else:
										SCREAMING_SNAKE_CASE_   :	List[Any]													=   logits.argmax(dim=1       )
										SCREAMING_SNAKE_CASE_   :	Any													=   [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]       )]

						return semantic_segmentation