
"""simple docstring"""






import itertools
import os
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed

import numpy as np

import datasets

from .execute import check_correctness


__SCREAMING_SNAKE_CASE							=     '\\n@misc{chen2021evaluating,\n      title={Evaluating Large Language Models Trained on Code},\n      author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \\nand Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \\nand Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \\nand Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \\nand Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \\nand Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \\nand Mohammad Bavarian and Clemens Winter and Philippe Tillet \\nand Felipe Petroski Such and Dave Cummings and Matthias Plappert \\nand Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \\nand William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \\nand Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \\nand William Saunders and Christopher Hesse and Andrew N. Carr \\nand Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \\nand Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \\nand Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \\nand Sam McCandlish and Ilya Sutskever and Wojciech Zaremba},\n      year={2021},\n      eprint={2107.03374},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG}\n}\n'

__SCREAMING_SNAKE_CASE							=     '\\nThis metric implements the evaluation harness for the HumanEval problem solving dataset\ndescribed in the paper \"Evaluating Large Language Models Trained on Code\"\n(https://arxiv.org/abs/2107.03374).\n'


__SCREAMING_SNAKE_CASE							=     '\nCalculates how good are predictions given some references, using certain scores\nArgs:\n    predictions: list of candidates to evaluate. Each candidates should be a list\n        of strings with several code candidates to solve the problem.\n    references: a list with a test for each prediction. Each test should evaluate the\n        correctness of a code candidate.\n    k: number of code candidates to consider in the evaluation (Default: [1, 10, 100])\n    num_workers: number of workers used to evaluate the canidate programs (Default: 4).\n    timeout:\nReturns:\n    pass_at_k: dict with pass rates for each k\n    results: dict with granular results of each unittest\nExamples:\n    >>> code_eval = datasets.load_metric(\"code_eval\")\n    >>> test_cases = [\"assert add(2,3)==5\"]\n    >>> candidates = [[\"def add(a,b): return a*b\", \"def add(a, b): return a+b\"]]\n    >>> pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2])\n    >>> print(pass_at_k)\n    {\'pass@1\': 0.5, \'pass@2\': 1.0}\n'


__SCREAMING_SNAKE_CASE							=     '\n################################################################################\n                                  !!!WARNING!!!\n################################################################################\nThe \"code_eval\" metric executes untrusted model-generated code in Python.\nAlthough it is highly unlikely that model-generated code will do something\novertly malicious in response to this test suite, model-generated code may act\ndestructively due to a lack of model capability or alignment.\nUsers are strongly encouraged to sandbox this evaluation suite so that it\ndoes not perform destructive actions on their host or network. For more\ninformation on how OpenAI sandboxes its code, see the paper \"Evaluating Large\nLanguage Models Trained on Code\" (https://arxiv.org/abs/2107.03374).\n\nOnce you have read this disclaimer and taken appropriate precautions,\nset the environment variable HF_ALLOW_CODE_EVAL=\"1\". Within Python you can to this\nwith:\n\n>>> import os\n>>> os.environ[\"HF_ALLOW_CODE_EVAL\"] = \"1\"\n\n################################################################################\\n'

__SCREAMING_SNAKE_CASE							=     'The MIT License\n\nCopyright (c) OpenAI (https://openai.com)\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.'





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,							_KWARGS_DESCRIPTION       )
class 				a__					(						datasets.Metric       ):



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      return datasets.MetricInfo(
          # This is the description that will appear on the metrics page.
          description=_DESCRIPTION      ,  citation=_CITATION      ,  inputs_description=_KWARGS_DESCRIPTION      ,  features=datasets.Features(
              {
                  'predictions': datasets.Sequence(datasets.Value('string'   )   ),
                  'references': datasets.Value('string'   ),
              }   )      ,  homepage='https://github.com/openai/human-eval'      ,  codebase_urls=['https://github.com/openai/human-eval']      ,  reference_urls=['https://github.com/openai/human-eval']      ,  license=_LICENSE      ,  )




     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :Union[str, Any]      ,  _lowerCamelCase :int      ,  _lowerCamelCase :List[Any]=[1, 10, 100]      ,  _lowerCamelCase :str=4      ,  _lowerCamelCase :Tuple=3.0   ):







      '''simple docstring'''



      if os.getenv('HF_ALLOW_CODE_EVAL'      ,  0   ) != "1":
       raise ValueError(_WARNING   )

      if os.name == "nt":
       raise NotImplementedError('This metric is currently not supported on Windows.'   )

      with ThreadPoolExecutor(max_workers=__lowerCamelCase   ) as executor:
       UpperCamelCase_     :					List[Any]   	=[]
       UpperCamelCase_     :					List[Any]   	=Counter()
       UpperCamelCase_     :					Any   	=0
       UpperCamelCase_     :					Optional[int]   	=defaultdict(__lowerCamelCase   )

       for task_id, (candidates, test_case) in enumerate(zip(__lowerCamelCase      ,  __lowerCamelCase   )   ):
        for candidate in candidates:
         UpperCamelCase_     :					str   	=candidate + '\n' + test_case
         UpperCamelCase_     :					Optional[Any]   	=(test_program, timeout, task_id, completion_id[task_id])
         UpperCamelCase_     :					Optional[Any]   	=executor.submit(__lowerCamelCase      ,  *__lowerCamelCase   )
         futures.append(__lowerCamelCase   )
         completion_id[task_id] += 1
         n_samples += 1

       for future in as_completed(__lowerCamelCase   ):
        UpperCamelCase_     :					List[str]   	=future.result()
        results[result["task_id"]].append((result['completion_id'], result)   )

      UpperCamelCase_	,     UpperCamelCase_     :					Optional[Any]   	=[], []
      for result in results.values():
       result.sort()
       UpperCamelCase_     :					Any   	=[r[1]['passed'] for r in result]
       total.append(len(__lowerCamelCase   )   )
       correct.append(sum(__lowerCamelCase   )   )
      UpperCamelCase_     :					Union[str, Any]   	=np.array(__lowerCamelCase   )
      UpperCamelCase_     :					List[str]   	=np.array(__lowerCamelCase   )

      UpperCamelCase_     :					Union[str, Any]   	=k
      UpperCamelCase_     :					Any   	={f'''pass@{k}''': estimate_pass_at_k(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase   ).mean() for k in ks if (total >= k).all()}

      return pass_at_k, results




def       A_  (				__lowercase						,	__lowercase						,	__lowercase	):

 def estimator(__lowercase						,	__lowercase						,	__lowercase	) -> float:
  if n - c < k:
   return 1.0
  return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1						,	n + 1	)	)

 if isinstance(lowerCAmelCase_						,	lowerCAmelCase_	):
  UpperCamelCase_     :					Optional[Any]   	=itertools.repeat(lowerCAmelCase_						,	len(lowerCAmelCase_	)	)
 else:
  assert len(lowerCAmelCase_	) == len(lowerCAmelCase_	)
  UpperCamelCase_     :					List[Any]   	=iter(lowerCAmelCase_	)

 return np.array([estimator(int(lowerCAmelCase_	)						,	int(lowerCAmelCase_	)						,	lowerCAmelCase_	) for n, c in zip(lowerCAmelCase_						,	lowerCAmelCase_	)]	)




import copy
import inspect
import unittest

from transformers import PretrainedConfig, SwiftFormerConfig
from transformers.testing_utils import (
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import SwiftFormerForImageClassification, SwiftFormerModel
 from transformers.models.swiftformer.modeling_swiftformer import SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import ViTImageProcessor






class 	__lowercase					:





      def __init__(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Any       ,							__lowerCamelCase    :       int=1_3       ,							__lowerCamelCase    :       List[str]=3       ,							__lowerCamelCase    :       List[str]=True       ,							__lowerCamelCase    :       Any=True       ,							__lowerCamelCase    :       Optional[int]=0.1       ,							__lowerCamelCase    :       Optional[Any]=0.1       ,							__lowerCamelCase    :       Tuple=2_2_4       ,							__lowerCamelCase    :       int=1_0_0_0       ,							__lowerCamelCase    :       int=[3, 3, 6, 4]       ,							__lowerCamelCase    :       Dict=[4_8, 5_6, 1_1_2, 2_2_0]       ,							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  parent
         UpperCAmelCase      					=  batch_size
         UpperCAmelCase      					=  num_channels
         UpperCAmelCase      					=  is_training
         UpperCAmelCase      					=  use_labels
         UpperCAmelCase      					=  hidden_dropout_prob
         UpperCAmelCase      					=  attention_probs_dropout_prob
         UpperCAmelCase      					=  num_labels
         UpperCAmelCase      					=  image_size
         UpperCAmelCase      					=  layer_depths
         UpperCAmelCase      					=  embed_dims





      def 							_lowercase					(		self    :       List[str]							)     -> Optional[Any]:






         """simple docstring"""







         UpperCAmelCase      					=  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

         UpperCAmelCase      					=  None
         if self.use_labels:
            UpperCAmelCase      					=  ids_tensor([self.batch_size]       ,							self.num_labels							)

         UpperCAmelCase      					=  self.get_config()

         return config, pixel_values, labels





      def 							_lowercase					(		self    :       Optional[Any]							)     -> str:






         """simple docstring"""







         return SwiftFormerConfig(
             depths=self.layer_depths       ,							embed_dims=self.embed_dims       ,							mlp_ratio=4       ,							downsamples=[True, True, True, True]       ,							hidden_act="""gelu"""       ,							num_labels=self.num_labels       ,							down_patch_size=3       ,							down_stride=2       ,							down_pad=1       ,							drop_rate=0.0       ,							drop_path_rate=0.0       ,							use_layer_scale=__lowerCamelCase       ,							layer_scale_init_value=1e-5       ,							)





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       int       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       Optional[int]							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  SwiftFormerModel(config=__lowerCamelCase							)
         model.to(__lowerCamelCase							)
         model.eval()
         UpperCAmelCase      					=  model(__lowerCamelCase							)
         self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.embed_dims[-1], 7, 7)							)





      def 							_lowercase					(		self    :       List[str]       ,							__lowerCamelCase    :       int       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       Any							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  self.num_labels
         UpperCAmelCase      					=  SwiftFormerForImageClassification(__lowerCamelCase							)
         model.to(__lowerCamelCase							)
         model.eval()
         UpperCAmelCase      					=  model(__lowerCamelCase       ,							labels=__lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_labels)							)

         UpperCAmelCase      					=  SwiftFormerForImageClassification(__lowerCamelCase							)
         model.to(__lowerCamelCase							)
         model.eval()

         UpperCAmelCase      					=  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)
         UpperCAmelCase      					=  model(__lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_labels)							)





      def 							_lowercase					(		self    :       Tuple							)     -> Optional[Any]:






         """simple docstring"""







         ((UpperCAmelCase)					,	(UpperCAmelCase)					,	(UpperCAmelCase))      					=  self.prepare_config_and_inputs()
         UpperCAmelCase      					=  {"""pixel_values""": pixel_values}
         return config, inputs_dict






@require_torch
class 	__lowercase					(	__snake_case			,       __snake_case			,       unittest.TestCase	):
      UpperCamelCase										=							(SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else ()
      UpperCamelCase										=							(
          {'''feature-extraction''': SwiftFormerModel, '''image-classification''': SwiftFormerForImageClassification}
          if is_torch_available()
          else {}
      )

      UpperCamelCase										=							False
      UpperCamelCase										=							False
      UpperCamelCase										=							False
      UpperCamelCase										=							False
      UpperCamelCase										=							False





      def 							_lowercase					(		self    :       str							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  SwiftFormerModelTester(self							)
         UpperCAmelCase      					=  ConfigTester(
             self       ,							config_class=__lowerCamelCase       ,							has_text_modality=__lowerCamelCase       ,							hidden_size=3_7       ,							num_attention_heads=1_2       ,							num_hidden_layers=1_2       ,							)





      def 							_lowercase					(		self    :       Dict							)     -> Union[str, Any]:






         """simple docstring"""







         self.config_tester.run_common_tests()





      @unittest.skip(reason="""SwiftFormer does not use inputs_embeds"""							)
      def 							_lowercase					(		self    :       Optional[int]							)     -> List[str]:






         """simple docstring"""







         pass





      def 							_lowercase					(		self    :       str							)     -> str:






         """simple docstring"""







         UpperCAmelCase					,	UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase      					=  model_class(__lowerCamelCase							)
            UpperCAmelCase      					=  model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(__lowerCamelCase       ,							nn.Linear							)							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> List[Any]:






         """simple docstring"""







         UpperCAmelCase					,	UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase      					=  model_class(__lowerCamelCase							)
            UpperCAmelCase      					=  inspect.signature(model.forward							)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            UpperCAmelCase      					=  [*signature.parameters.keys()]

            UpperCAmelCase      					=  ["""pixel_values"""]
            self.assertListEqual(arg_names[:1]       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       List[str]							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__lowerCamelCase							)





      def 							_lowercase					(		self    :       Optional[Any]							)     -> List[Any]:






         """simple docstring"""







         UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*__lowerCamelCase							)





      @slow
      def 							_lowercase					(		self    :       Optional[Any]							)     -> Optional[Any]:






         """simple docstring"""







         for model_name in SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCAmelCase      					=  SwiftFormerModel.from_pretrained(__lowerCamelCase							)
            self.assertIsNotNone(__lowerCamelCase							)





      @unittest.skip(reason="""SwiftFormer does not output attentions"""							)
      def 							_lowercase					(		self    :       List[str]							)     -> Union[str, Any]:






         """simple docstring"""







         pass





      def 							_lowercase					(		self    :       Dict							)     -> List[Any]:






         """simple docstring"""







         def check_hidden_states_output(__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Optional[Any]							):
            UpperCAmelCase      					=  model_class(__lowerCamelCase							)
            model.to(__lowerCamelCase							)
            model.eval()

            with torch.no_grad():
               UpperCAmelCase      					=  model(**self._prepare_for_class(__lowerCamelCase       ,							__lowerCamelCase							)							)

            UpperCAmelCase      					=  outputs.hidden_states

            UpperCAmelCase      					=  8
            self.assertEqual(len(__lowerCamelCase							)       ,							__lowerCamelCase							)  # TODO

            # SwiftFormer's feature maps are of shape (batch_size, embed_dims, height, width)
            # with the width and height being successively divided by 2, after every 2 blocks
            for i in range(len(__lowerCamelCase							)							):
               self.assertEqual(
                   hidden_states[i].shape       ,							torch.Size(
                       [
                           self.model_tester.batch_size,
                           self.model_tester.embed_dims[i // 2],
                           (self.model_tester.image_size // 4) // 2 ** (i // 2),
                           (self.model_tester.image_size // 4) // 2 ** (i // 2),
                       ]							)       ,							)

         UpperCAmelCase					,	UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            UpperCAmelCase      					=  True
            check_hidden_states_output(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            UpperCAmelCase      					=  True

            check_hidden_states_output(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       Dict							)     -> Union[str, Any]:






         """simple docstring"""







         def _config_zero_init(__lowerCamelCase    :       Any							):
            UpperCAmelCase      					=  copy.deepcopy(__lowerCamelCase							)
            for key in configs_no_init.__dict__.keys():
               if "_range" in key or "_std" in key or "initializer_factor" in key or "layer_scale" in key:
                  setattr(__lowerCamelCase       ,							__lowerCamelCase       ,							1e-1_0							)
               if isinstance(getattr(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)       ,							__lowerCamelCase							):
                  UpperCAmelCase      					=  _config_zero_init(getattr(__lowerCamelCase       ,							__lowerCamelCase							)							)
                  setattr(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)
            return configs_no_init

         UpperCAmelCase					,	UpperCAmelCase      					=  self.model_tester.prepare_config_and_inputs_for_common()

         UpperCAmelCase      					=  _config_zero_init(__lowerCamelCase							)
         for model_class in self.all_model_classes:
            UpperCAmelCase      					=  model_class(config=__lowerCamelCase							)
            for name, param in model.named_parameters():
               if param.requires_grad:
                  self.assertIn(
                      ((param.data.mean() * 1e9) / 1e9).round().item()       ,							[0.0, 1.0]       ,							msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""       ,							)





      @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""							)
      def 							_lowercase					(		self    :       Optional[int]							)     -> List[str]:






         """simple docstring"""







         pass

def 	_UpperCamelCase					(    )    ->Dict:
   UpperCAmelCase      					=  Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""" )
   return image







@require_torch
@require_vision
class 	__lowercase					(	unittest.TestCase	):





      @cached_property
      def 							_lowercase					(		self    :       Optional[int]							)     -> int:






         """simple docstring"""







         return ViTImageProcessor.from_pretrained("""MBZUAI/swiftformer-xs"""							) if is_vision_available() else None





      @slow
      def 							_lowercase					(		self    :       Optional[int]							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  SwiftFormerForImageClassification.from_pretrained("""MBZUAI/swiftformer-xs"""							).to(__lowerCamelCase							)

         UpperCAmelCase      					=  self.default_image_processor
         UpperCAmelCase      					=  prepare_img()
         UpperCAmelCase      					=  image_processor(images=__lowerCamelCase       ,							return_tensors="""pt"""							).to(__lowerCamelCase							)

         # forward pass
         with torch.no_grad():
            UpperCAmelCase      					=  model(**__lowerCamelCase							)

         # verify the logits
         UpperCAmelCase      					=  torch.Size((1, 1_0_0_0)							)
         self.assertEqual(outputs.logits.shape       ,							__lowerCamelCase							)

         UpperCAmelCase      					=  torch.tensor([[-2.1_7_0_3e0_0, 2.1_1_0_7e0_0, -2.0_8_1_1e0_0]]							).to(__lowerCamelCase							)
         self.assertTrue(torch.allclose(outputs.logits[0, :3]       ,							__lowerCamelCase       ,							atol=1e-4							)							)



