




'''simple docstring'''


from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCAmelCase_					=   logging.get_logger(__name__)


UpperCAmelCase_					=   {
    "Salesforce/codegen-350M-nl": "https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json",
    "Salesforce/codegen-350M-multi": "https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json",
    "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json",
    "Salesforce/codegen-2B-nl": "https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json",
    "Salesforce/codegen-2B-multi": "https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json",
    "Salesforce/codegen-2B-mono": "https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json",
    "Salesforce/codegen-6B-nl": "https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json",
    "Salesforce/codegen-6B-multi": "https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json",
    "Salesforce/codegen-6B-mono": "https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json",
    "Salesforce/codegen-16B-nl": "https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json",
    "Salesforce/codegen-16B-multi": "https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json",
    "Salesforce/codegen-16B-mono": "https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json",
}





class  __lowercase    (	__magic_name__			):
   _a    						=   """codegen"""
   _a    						=   {
       """max_position_embeddings""": """n_positions""",
       """hidden_size""": """n_embd""",
       """num_attention_heads""": """n_head""",
       """num_hidden_layers""": """n_layer""",
   }







   def __init__(						self   ,				UpperCamelCase=5_0400   ,				UpperCamelCase=2048   ,				UpperCamelCase=2048   ,				UpperCamelCase=4096   ,				UpperCamelCase=28   ,				UpperCamelCase=16   ,				UpperCamelCase=64   ,				UpperCamelCase=None   ,				UpperCamelCase="gelu_new"   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.0   ,				UpperCamelCase=1e-5   ,				UpperCamelCase=0.02   ,				UpperCamelCase=True   ,				UpperCamelCase=5_0256   ,				UpperCamelCase=5_0256   ,				UpperCamelCase=False   ,				**UpperCamelCase   ,				)					->     List[str]:
     __a								=       vocab_size
     __a								=       n_ctx
     __a								=       n_positions
     __a								=       n_embd
     __a								=       n_layer
     __a								=       n_head
     __a								=       n_inner
     __a								=       rotary_dim
     __a								=       activation_function
     __a								=       resid_pdrop
     __a								=       embd_pdrop
     __a								=       attn_pdrop
     __a								=       layer_norm_epsilon
     __a								=       initializer_range
     __a								=       use_cache

     __a								=       bos_token_id
     __a								=       eos_token_id

     super().__init__(
         bos_token_id=UpperCamelCase   ,				eos_token_id=UpperCamelCase   ,				tie_word_embeddings=UpperCamelCase   ,				**UpperCamelCase   )





class  __lowercase    (	__magic_name__			):







   def __init__(						self   ,				UpperCamelCase   ,				UpperCamelCase = "default"   ,				UpperCamelCase = None   ,				UpperCamelCase = False   ,				)					->     List[Any]:
     super().__init__(UpperCamelCase   ,				task=UpperCamelCase   ,				patching_specs=UpperCamelCase   ,				use_past=UpperCamelCase   )
     if not getattr(self._config   ,				'pad_token_id'   ,				UpperCamelCase   ):
       # TODO: how to do that better?
       __a								=       0







   @property
   def 		UpperCamelCase__      (						self   )					->     Mapping[str, Mapping[int, str]]:
     __a								=       OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
     if self.use_past:
       self.fill_with_past_key_values_(UpperCamelCase   ,				direction='inputs'   )
       __a								=       {0: 'batch', 1: 'past_sequence + sequence'}
     else:
       __a								=       {0: 'batch', 1: 'sequence'}

     return common_inputs







   @property
   def 		UpperCamelCase__      (						self   )					->     int:
     return self._config.n_layer







   @property
   def 		UpperCamelCase__      (						self   )					->     int:
     return self._config.n_head







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase = -1   ,				UpperCamelCase = -1   ,				UpperCamelCase = False   ,				UpperCamelCase = None   ,				)					->     Mapping[str, Any]:
     __a								=       super(UpperCamelCase   ,				self   ).generate_dummy_inputs(
         UpperCamelCase   ,				batch_size=UpperCamelCase   ,				seq_length=UpperCamelCase   ,				is_pair=UpperCamelCase   ,				framework=UpperCamelCase   )

     # We need to order the input in the way they appears in the forward()
     __a								=       OrderedDict({'input_ids': common_inputs['input_ids']}   )

     # Need to add the past_keys
     if self.use_past:
       if not is_torch_available():
         raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
       else:
         import torch

         __a						,							__a								=       common_inputs['input_ids'].shape
         # Not using the same length for past_key_values
         __a								=       seqlen + 2
         __a								=       (
             batch,
             self.num_attention_heads,
             past_key_values_length,
             self._config.hidden_size // self.num_attention_heads,
         )
         __a								=       [
             (torch.zeros(UpperCamelCase   ), torch.zeros(UpperCamelCase   )) for _ in range(self.num_layers   )
         ]

     __a								=       common_inputs['attention_mask']
     if self.use_past:
       __a								=       ordered_inputs['attention_mask'].dtype
       __a								=       torch.cat(
           [ordered_inputs['attention_mask'], torch.ones(UpperCamelCase   ,				UpperCamelCase   ,				dtype=UpperCamelCase   )]   ,				dim=1   )

     return ordered_inputs







   @property
   def 		UpperCamelCase__      (						self   )					->     int:
     return 13









'''simple docstring'''


import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment





@require_huggingface_suite
class  __lowercase    (	unittest.TestCase			):







   def 		UpperCamelCase__      (						self   )					->     Tuple:
     __a								=       inspect.getfile(accelerate.test_utils   )
     __a								=       os.path.sep.join(
         mod_file.split(os.path.sep   )[:-1] + ['scripts', 'external_deps', 'test_metrics.py']   )

     from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

     __a								=       test_metrics







   @require_cpu
   def 		UpperCamelCase__      (						self   )					->     Tuple:
     debug_launcher(self.test_metrics.main   ,				num_processes=1   )







   @require_cpu
   def 		UpperCamelCase__      (						self   )					->     str:
     debug_launcher(self.test_metrics.main   )







   @require_single_gpu
   def 		UpperCamelCase__      (						self   )					->     Dict:
     self.test_metrics.main()







   @require_multi_gpu
   def 		UpperCamelCase__      (						self   )					->     int:
     print(f"Found {torch.cuda.device_count()} devices."   )
     __a								=       ['torchrun', f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
     with patch_environment(omp_num_threads=1   ):
       execute_subprocess_async(UpperCamelCase   ,				env=os.environ.copy()   )



