





'''simple docstring'''



import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__magic_name__   :						Optional[Any]    =			["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]




class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     def __init__(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase=None						,   lowerCamelCase=1	):
       _snake_case     		=				tokenizer
       _snake_case     		=				dataset
       _snake_case     		=				len(lowerCamelCase	) if n_tasks is None else n_tasks
       _snake_case     		=				n_copies






     def __iter__(					self	):
       _snake_case     		=				[]
       for task in range(self.n_tasks	):
         # without strip, the model generate commented codes ...
         prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()	)
       _snake_case     		=				self.tokenizer(lowerCamelCase						,   padding=lowerCamelCase						,   return_tensors="pt"	)
       for task in range(self.n_tasks	):
         for _ in range(self.n_copies	):
           yield {
               "ids": outputs.input_ids[task],
               "task_id": task,
               "input_len": outputs.attention_mask[task].sum(),
           }







class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     def __init__(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase	):
       _snake_case     		=				start_length
       _snake_case     		=				eof_strings
       _snake_case     		=				tokenizer






     def __call__(					self						,   lowerCamelCase						,   lowerCamelCase						,   **lowerCamelCase	):
       _snake_case     		=				self.tokenizer.batch_decode(input_ids[:, self.start_length :]	)
       _snake_case     		=				[]
       for decoded_generation in decoded_generations:
         done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings	)	)
       return all(lowerCamelCase	)







def 				snake_case_      (		SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  _snake_case     		=				re.split("(%s)" % "|".join(SCREAMING_SNAKE_CASE__       )					,	SCREAMING_SNAKE_CASE__       )
  # last string should be ""
  return "".join(string_list[:-2]       )




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__=20					,	**SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  _snake_case     		=				defaultdict(SCREAMING_SNAKE_CASE__       )  # dict of list of generated tokens
  for step, batch in tqdm(enumerate(SCREAMING_SNAKE_CASE__       )       ):
    with torch.no_grad():
      _snake_case     		=				batch["ids"].shape[-1]
      _snake_case     		=				accelerator.unwrap_model(SCREAMING_SNAKE_CASE__       ).generate(
          input_ids=batch["ids"][:, : batch["input_len"]]					,	num_return_sequences=SCREAMING_SNAKE_CASE__					,	**SCREAMING_SNAKE_CASE__       )
      # each task is generated batch_size times
      _snake_case     		=				batch["task_id"].repeat(SCREAMING_SNAKE_CASE__       )
      _snake_case     		=				accelerator.pad_across_processes(
          SCREAMING_SNAKE_CASE__					,	dim=1					,	pad_index=tokenizer.pad_token_id       )

      _snake_case		,		_snake_case     		=				accelerator.gather((generated_tokens, generated_tasks)       )
      _snake_case     		=				generated_tokens.cpu().numpy()
      _snake_case     		=				generated_tasks.cpu().numpy()

      for task, generated_tokens in zip(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):
        gen_token_dict[task].append(SCREAMING_SNAKE_CASE__       )

  _snake_case     		=				[[] for _ in range(SCREAMING_SNAKE_CASE__       )]
  for task, generated_tokens in gen_token_dict.items():
    for s in generated_tokens:
      _snake_case     		=				tokenizer.decode(SCREAMING_SNAKE_CASE__					,	skip_special_tokens=SCREAMING_SNAKE_CASE__					,	clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__       )
      code_gens[task].append(remove_last_block(SCREAMING_SNAKE_CASE__       )       )
  return code_gens




def 				snake_case_      (		):







  '''simple docstring'''



  _snake_case     		=				HfArgumentParser(SCREAMING_SNAKE_CASE__       )
  _snake_case     		=				parser.parse_args()

  transformers.logging.set_verbosity_error()
  # enables code execution in code_eval metric
  _snake_case     		=				args.HF_ALLOW_CODE_EVAL
  # make sure tokenizer plays nice with multiprocessing
  _snake_case     		=				"false"

  if args.num_workers is None:
    _snake_case     		=				multiprocessing.cpu_count()

  # Use dataset load to feed to accelerate
  _snake_case     		=				Accelerator()
  set_seed(args.seed					,	device_specific=SCREAMING_SNAKE_CASE__       )

  # Load model and tokenizer
  _snake_case     		=				AutoTokenizer.from_pretrained(args.model_ckpt       )
  _snake_case     		=				tokenizer.eos_token
  _snake_case     		=				AutoModelForCausalLM.from_pretrained(args.model_ckpt       )

  # Generation settings
  _snake_case     		=				{
      "do_sample": args.do_sample,
      "temperature": args.temperature,
      "max_new_tokens": args.max_new_tokens,
      "top_p": args.top_p,
      "top_k": args.top_k,
      "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )]       ),
  }

  # Load evaluation dataset and metric
  _snake_case     		=				load_dataset("openai_humaneval"       )
  _snake_case     		=				load_metric("code_eval"       )

  _snake_case     		=				args.num_tasks if args.num_tasks is not None else len(human_eval["test"]       )
  _snake_case     		=				args.n_samples // args.batch_size

  _snake_case     		=				TokenizedDataset(SCREAMING_SNAKE_CASE__					,	human_eval["test"]					,	n_copies=SCREAMING_SNAKE_CASE__					,	n_tasks=SCREAMING_SNAKE_CASE__       )
  # do not confuse args.batch_size, which is actually the num_return_sequences
  _snake_case     		=				DataLoader(SCREAMING_SNAKE_CASE__					,	batch_size=1       )

  # Run a quick test to see if code evaluation is enabled
  try:
    _snake_case     		=				code_eval_metric.compute(references=[""]					,	predictions=[[""]]       )
  except ValueError as exception:
    print(
        "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
        " flag to enable code evaluation."       )
    raise exception

  _snake_case		,		_snake_case     		=				accelerator.prepare(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )

  _snake_case     		=				complete_code(
      SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	n_tasks=SCREAMING_SNAKE_CASE__					,	batch_size=args.batch_size					,	**SCREAMING_SNAKE_CASE__					,	)

  if accelerator.is_main_process:
    _snake_case     		=				[]

    for task in tqdm(range(SCREAMING_SNAKE_CASE__       )       ):
      _snake_case     		=				human_eval["test"][task]["test"]
      _snake_case     		=				f'''check({human_eval["test"][task]["entry_point"]})'''
      references.append("\n" + test_func + "\n" + entry_point       )

    # Evaluate completions with "code_eval" metric
    _snake_case		,		_snake_case     		=				code_eval_metric.compute(
        references=SCREAMING_SNAKE_CASE__					,	predictions=SCREAMING_SNAKE_CASE__					,	num_workers=args.num_workers       )
    print(f'''Results: {pass_at_k}'''       )

    # Save results to json file
    with open(args.output_file					,	"w"       ) as fp:
      json.dump(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
 main()









'''simple docstring'''



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__magic_name__   :						int    =			logging.get_logger(__name__)

__magic_name__   :						Tuple    =			{
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}


class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						Union[str, Any]      =   '''xlm'''
     UpperCAmelCase__       :						Dict      =   {
         '''hidden_size''': '''emb_dim''',
         '''num_attention_heads''': '''n_heads''',
         '''num_hidden_layers''': '''n_layers''',
         '''n_words''': '''vocab_size''',  # For backward compatibility
     }

     def __init__(					self						,   lowerCamelCase=30_145						,   lowerCamelCase=2_048						,   lowerCamelCase=12						,   lowerCamelCase=16						,   lowerCamelCase=0.1						,   lowerCamelCase=0.1						,   lowerCamelCase=True						,   lowerCamelCase=False						,   lowerCamelCase=False						,   lowerCamelCase=False						,   lowerCamelCase=1						,   lowerCamelCase=True						,   lowerCamelCase=512						,   lowerCamelCase=2_048**-0.5						,   lowerCamelCase=1e-12						,   lowerCamelCase=0.02						,   lowerCamelCase=0						,   lowerCamelCase=1						,   lowerCamelCase=2						,   lowerCamelCase=3						,   lowerCamelCase=5						,   lowerCamelCase=True						,   lowerCamelCase="first"						,   lowerCamelCase=True						,   lowerCamelCase=None						,   lowerCamelCase=True						,   lowerCamelCase=0.1						,   lowerCamelCase=5						,   lowerCamelCase=5						,   lowerCamelCase=0						,   lowerCamelCase=0						,   lowerCamelCase=2						,   lowerCamelCase=0						,   **lowerCamelCase						,   ):
       _snake_case     		=				vocab_size
       _snake_case     		=				emb_dim
       _snake_case     		=				n_layers
       _snake_case     		=				n_heads
       _snake_case     		=				dropout
       _snake_case     		=				attention_dropout
       _snake_case     		=				gelu_activation
       _snake_case     		=				sinusoidal_embeddings
       _snake_case     		=				causal
       _snake_case     		=				asm
       _snake_case     		=				n_langs
       _snake_case     		=				use_lang_emb
       _snake_case     		=				layer_norm_eps
       _snake_case     		=				bos_index
       _snake_case     		=				eos_index
       _snake_case     		=				pad_index
       _snake_case     		=				unk_index
       _snake_case     		=				mask_index
       _snake_case     		=				is_encoder
       _snake_case     		=				max_position_embeddings
       _snake_case     		=				embed_init_std
       _snake_case     		=				init_std
       _snake_case     		=				summary_type
       _snake_case     		=				summary_use_proj
       _snake_case     		=				summary_activation
       _snake_case     		=				summary_proj_to_labels
       _snake_case     		=				summary_first_dropout
       _snake_case     		=				start_n_top
       _snake_case     		=				end_n_top
       _snake_case     		=				mask_token_id
       _snake_case     		=				lang_id

       if "n_words" in kwargs:
         _snake_case     		=				kwargs["n_words"]

       super().__init__(pad_token_id=lowerCamelCase						,   bos_token_id=lowerCamelCase						,   **lowerCamelCase	)







class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     @property
     def 	UpperCamelCase(					self	):
       if self.task == "multiple-choice":
         _snake_case     		=				{0: "batch", 1: "choice", 2: "sequence"}
       else:
         _snake_case     		=				{0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("input_ids", dynamic_axis),
               ("attention_mask", dynamic_axis),
               ("token_type_ids", dynamic_axis),
           ]	)


