

import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__lowercase				      =						["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]

class    _lowercase				(					__lowerCamelCase			):

   def __init__(				self	:    List[str]			,						lowerCamelCase__	:    Dict			,						lowerCamelCase__	:    Optional[Any]			,						lowerCamelCase__	:    Tuple=None			,						lowerCamelCase__	:    Union[str, Any]=1       ) ->			Optional[Any]:
         """simple docstring"""





         A_							       =		tokenizer
         A_							       =		dataset
         A_							       =		len(lowerCamelCase__       ) if n_tasks is None else n_tasks
         A_							       =		n_copies





   def __iter__(				self	:    str       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		[]
         for task in range(self.n_tasks       ):
               # without strip, the model generate commented codes ...
               prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()       )
         A_							       =		self.tokenizer(lowerCamelCase__			,						padding=lowerCamelCase__			,						return_tensors='''pt'''       )
         for task in range(self.n_tasks       ):
               for _ in range(self.n_copies       ):
                     yield {
                         "ids": outputs.input_ids[task],
                         "task_id": task,
                         "input_len": outputs.attention_mask[task].sum(),
                     }



class    _lowercase				(					__lowerCamelCase			):

   def __init__(				self	:    Union[str, Any]			,						lowerCamelCase__	:    str			,						lowerCamelCase__	:    Optional[int]			,						lowerCamelCase__	:    List[str]       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		start_length
         A_							       =		eof_strings
         A_							       =		tokenizer





   def __call__(				self	:    Dict			,						lowerCamelCase__	:    Dict			,						lowerCamelCase__	:    List[Any]			,						**lowerCamelCase__	:    Union[str, Any]       ) ->			Dict:
         """simple docstring"""





         A_							       =		self.tokenizer.batch_decode(input_ids[:, self.start_length :]       )
         A_							       =		[]
         for decoded_generation in decoded_generations:
               done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings       )       )
         return all(lowerCamelCase__       )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		re.split('''(%s)''' % '''|'''.join(SCREAMING_SNAKE_CASE   )		,				SCREAMING_SNAKE_CASE   )
      # last string should be ""
      return "".join(string_list[:-2]   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=20		,				**SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		defaultdict(SCREAMING_SNAKE_CASE   )  # dict of list of generated tokens
      for step, batch in tqdm(enumerate(SCREAMING_SNAKE_CASE   )   ):
            with torch.no_grad():
                  A_							       =		batch['''ids'''].shape[-1]
                  A_							       =		accelerator.unwrap_model(SCREAMING_SNAKE_CASE   ).generate(
                      input_ids=batch['''ids'''][:, : batch['''input_len''']]		,				num_return_sequences=SCREAMING_SNAKE_CASE		,				**SCREAMING_SNAKE_CASE   )
                  # each task is generated batch_size times
                  A_							       =		batch['''task_id'''].repeat(SCREAMING_SNAKE_CASE   )
                  A_							       =		accelerator.pad_across_processes(
                      SCREAMING_SNAKE_CASE		,				dim=1		,				pad_index=tokenizer.pad_token_id   )

                  A_       ,A_							       =		accelerator.gather((generated_tokens, generated_tasks)   )
                  A_							       =		generated_tokens.cpu().numpy()
                  A_							       =		generated_tasks.cpu().numpy()

                  for task, generated_tokens in zip(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
                        gen_token_dict[task].append(SCREAMING_SNAKE_CASE   )

      A_							       =		[[] for _ in range(SCREAMING_SNAKE_CASE   )]
      for task, generated_tokens in gen_token_dict.items():
            for s in generated_tokens:
                  A_							       =		tokenizer.decode(SCREAMING_SNAKE_CASE		,				skip_special_tokens=SCREAMING_SNAKE_CASE		,				clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE   )
                  code_gens[task].append(remove_last_block(SCREAMING_SNAKE_CASE   )   )
      return code_gens







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		HfArgumentParser(SCREAMING_SNAKE_CASE   )
      A_							       =		parser.parse_args()

      transformers.logging.set_verbosity_error()
      # enables code execution in code_eval metric
      A_							       =		args.HF_ALLOW_CODE_EVAL
      # make sure tokenizer plays nice with multiprocessing
      A_							       =		'''false'''

      if args.num_workers is None:
            A_							       =		multiprocessing.cpu_count()

      # Use dataset load to feed to accelerate
      A_							       =		Accelerator()
      set_seed(args.seed		,				device_specific=SCREAMING_SNAKE_CASE   )

      # Load model and tokenizer
      A_							       =		AutoTokenizer.from_pretrained(args.model_ckpt   )
      A_							       =		tokenizer.eos_token
      A_							       =		AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

      # Generation settings
      A_							       =		{
          '''do_sample''': args.do_sample,
          '''temperature''': args.temperature,
          '''max_new_tokens''': args.max_new_tokens,
          '''top_p''': args.top_p,
          '''top_k''': args.top_k,
          '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )]   ),
      }

      # Load evaluation dataset and metric
      A_							       =		load_dataset('''openai_humaneval'''   )
      A_							       =		load_metric('''code_eval'''   )

      A_							       =		args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']   )
      A_							       =		args.n_samples // args.batch_size

      A_							       =		TokenizedDataset(SCREAMING_SNAKE_CASE		,				human_eval['''test''']		,				n_copies=SCREAMING_SNAKE_CASE		,				n_tasks=SCREAMING_SNAKE_CASE   )
      # do not confuse args.batch_size, which is actually the num_return_sequences
      A_							       =		DataLoader(SCREAMING_SNAKE_CASE		,				batch_size=1   )

      # Run a quick test to see if code evaluation is enabled
      try:
            A_							       =		code_eval_metric.compute(references=['''''']		,				predictions=[['''''']]   )
      except ValueError as exception:
            print(
                '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
                ''' flag to enable code evaluation.'''   )
            raise exception

      A_       ,A_							       =		accelerator.prepare(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )

      A_							       =		complete_code(
          SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				n_tasks=SCREAMING_SNAKE_CASE		,				batch_size=args.batch_size		,				**SCREAMING_SNAKE_CASE		,				)

      if accelerator.is_main_process:
            A_							       =		[]

            for task in tqdm(range(SCREAMING_SNAKE_CASE   )   ):
                  A_							       =		human_eval['''test'''][task]['''test''']
                  A_							       =		f"check({human_eval['test'][task]['entry_point']})"
                  references.append('''\n''' + test_func + '''\n''' + entry_point   )

            # Evaluate completions with "code_eval" metric
            A_       ,A_							       =		code_eval_metric.compute(
                references=SCREAMING_SNAKE_CASE		,				predictions=SCREAMING_SNAKE_CASE		,				num_workers=args.num_workers   )
            print(f"Results: {pass_at_k}"   )

            # Save results to json file
            with open(args.output_file		,				'''w'''   ) as fp:
                  json.dump(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
      main()








import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
__lowercase				      =						"""src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
__lowercase				      =						direct_transformers_import(PATH_TO_TRANSFORMERS)

__lowercase				      =						transformers.models.auto.configuration_auto.CONFIG_MAPPING

__lowercase				      =						{
    # used to compute the property `self.chunk_length`
    """EncodecConfig""": ["""overlap"""],
    # used as `self.bert_model = BertModel(config, ...)`
    """DPRConfig""": True,
    # not used in modeling files, but it's an important information
    """FSMTConfig""": ["""langs"""],
    # used internally in the configuration class file
    """GPTNeoConfig""": ["""attention_types"""],
    # used internally in the configuration class file
    """EsmConfig""": ["""is_folding_model"""],
    # used during training (despite we don't have training script for these models yet)
    """Mask2FormerConfig""": ["""ignore_value"""],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    """OneFormerConfig""": ["""ignore_value""", """norm"""],
    # used during preprocessing and collation, see `collating_graphormer.py`
    """GraphormerConfig""": ["""spatial_pos_max"""],
    # used internally in the configuration class file
    """T5Config""": ["""feed_forward_proj"""],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    """MT5Config""": ["""feed_forward_proj""", """tokenizer_class"""],
    """UMT5Config""": ["""feed_forward_proj""", """tokenizer_class"""],
    # used internally in the configuration class file
    """LongT5Config""": ["""feed_forward_proj"""],
    # used internally in the configuration class file
    """SwitchTransformersConfig""": ["""feed_forward_proj"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """BioGptConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """GLPNConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """SegformerConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """CvtConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """PerceiverConfig""": ["""layer_norm_eps"""],
    # used internally to calculate the feature size
    """InformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate the feature size
    """TimeSeriesTransformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate the feature size
    """AutoformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate `mlp_dim`
    """SamVisionConfig""": ["""mlp_ratio"""],
    # For (head) training, but so far not implemented
    """ClapAudioConfig""": ["""num_classes"""],
    # Not used, but providing useful information to users
    """SpeechT5HifiGanConfig""": ["""sampling_rate"""],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        """CLIPSegConfig""": True,
        """DeformableDetrConfig""": True,
        """DetaConfig""": True,
        """DinatConfig""": True,
        """DonutSwinConfig""": True,
        """EfficientFormerConfig""": True,
        """FSMTConfig""": True,
        """JukeboxConfig""": True,
        """LayoutLMv2Config""": True,
        """MaskFormerSwinConfig""": True,
        """MT5Config""": True,
        """NatConfig""": True,
        """OneFormerConfig""": True,
        """PerceiverConfig""": True,
        """RagConfig""": True,
        """SpeechT5Config""": True,
        """SwinConfig""": True,
        """Swin2SRConfig""": True,
        """Swinv2Config""": True,
        """SwitchTransformersConfig""": True,
        """TableTransformerConfig""": True,
        """TapasConfig""": True,
        """TransfoXLConfig""": True,
        """UniSpeechConfig""": True,
        """UniSpeechSatConfig""": True,
        """WavLMConfig""": True,
        """WhisperConfig""": True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        """JukeboxPriorConfig""": True,
        # TODO: @Younes (for `is_decoder`)
        """Pix2StructTextConfig""": True,
    }
)







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		False
      for attribute in attributes:
            for modeling_source in source_strings:
                  # check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
                  if (
                      f"config.{attribute}" in modeling_source
                      or f"getattr(config, \"{attribute}\"" in modeling_source
                      or f"getattr(self.config, \"{attribute}\"" in modeling_source
                  ):
                        A_							       =		True
                  # Deal with multi-line cases
                  elif (
                      re.search(
                          Rf"getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*\"{attribute}\""		,				SCREAMING_SNAKE_CASE		,				)
                      is not None
                  ):
                        A_							       =		True
                  # `SequenceSummary` is called with `SequenceSummary(config)`
                  elif attribute in [
                      "summary_type",
                      "summary_use_proj",
                      "summary_activation",
                      "summary_last_dropout",
                      "summary_proj_to_labels",
                      "summary_first_dropout",
                  ]:
                        if "SequenceSummary" in modeling_source:
                              A_							       =		True
                  if attribute_used:
                        break
            if attribute_used:
                  break

    # common and important attributes, even if they do not always appear in the modeling files
      A_							       =		[
          '''bos_index''',
          '''eos_index''',
          '''pad_index''',
          '''unk_index''',
          '''mask_index''',
          '''image_size''',
          '''use_cache''',
          '''out_features''',
          '''out_indices''',
      ]
      A_							       =		['''encoder_no_repeat_ngram_size''']

      # Special cases to be allowed
      A_							       =		True
      if not attribute_used:
            A_							       =		False
            for attribute in attributes:
                  # Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
                  if attribute in ["is_encoder_decoder"] and default_value is True:
                        A_							       =		True
                  elif attribute in ["tie_word_embeddings"] and default_value is False:
                        A_							       =		True

                  # Allow cases without checking the default value in the configuration class
                  elif attribute in attributes_to_allow + attributes_used_in_generation:
                        A_							       =		True
                  elif attribute.endswith('''_token_id'''   ):
                        A_							       =		True

                  # configuration class specific cases
                  if not case_allowed:
                        A_							       =		SPECIAL_CASES_TO_ALLOW.get(config_class.__name__		,				[]   )
                        A_							       =		allowed_cases is True or attribute in allowed_cases

      return attribute_used or case_allowed







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		dict(inspect.signature(config_class.__init__   ).parameters   )
      A_							       =		[x for x in list(signature.keys()   ) if x not in ['''self''', '''kwargs''']]
      A_							       =		[signature[param].default for param in parameter_names]

      # If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
      # as one variant is used, the test should pass
      A_							       =		{}
      if len(config_class.attribute_map   ) > 0:
            A_							       =		{v: k for k, v in config_class.attribute_map.items()}

      # Get the path to modeling source files
      A_							       =		inspect.getsourcefile(SCREAMING_SNAKE_CASE   )
      A_							       =		os.path.dirname(SCREAMING_SNAKE_CASE   )
      # Let's check against all frameworks: as long as one framework uses an attribute, we are good.
      A_							       =		[os.path.join(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ) for fn in os.listdir(SCREAMING_SNAKE_CASE   ) if fn.startswith('''modeling_'''   )]

      # Get the source code strings
      A_							       =		[]
      for path in modeling_paths:
            if os.path.isfile(SCREAMING_SNAKE_CASE   ):
                  with open(SCREAMING_SNAKE_CASE   ) as fp:
                        modeling_sources.append(fp.read()   )

      A_							       =		[]
      for config_param, default_value in zip(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
            # `attributes` here is all the variant names for `config_param`
            A_							       =		[config_param]
            # some configuration classes have non-empty `attribute_map`, and both names could be used in the
            # corresponding modeling files. As long as one of them appears, it is fine.
            if config_param in reversed_attribute_map:
                  attributes.append(reversed_attribute_map[config_param]   )

            if not check_attribute_being_used(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
                  unused_attributes.append(attributes[0]   )

      return sorted(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		{}
      for _config_class in list(CONFIG_MAPPING.values()   ):
            # Skip deprecated models
            if "models.deprecated" in _config_class.__module__:
                  continue
            # Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
            A_							       =		[
                cls
                for name, cls in inspect.getmembers(
                    inspect.getmodule(_config_class   )		,				lambda SCREAMING_SNAKE_CASE   : inspect.isclass(SCREAMING_SNAKE_CASE   )
                    and issubclass(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                    and inspect.getmodule(SCREAMING_SNAKE_CASE   ) == inspect.getmodule(_config_class   )		,				)
            ]
            for config_class in config_classes_in_module:
                  A_							       =		check_config_attributes_being_used(SCREAMING_SNAKE_CASE   )
                  if len(SCREAMING_SNAKE_CASE   ) > 0:
                        A_							       =		unused_attributes

      if len(SCREAMING_SNAKE_CASE   ) > 0:
            A_							       =		'''The following configuration classes contain unused attributes in the corresponding modeling files:\n'''
            for name, attributes in configs_with_unused_attributes.items():
                  error += f"{name}: {attributes}\n"

            raise ValueError(SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
      check_config_attributes()





