


"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCamelCase__				:Tuple			   =	logging.get_logger(__name__)

UpperCamelCase__				:Dict			   =	{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}






class      A(  UpperCAmelCase__		):

		"""simple docstring"""





		A    =							"gptj"
		A    =							{
		    "max_position_embeddings": "n_positions",
		    "hidden_size": "n_embd",
		    "num_attention_heads": "n_head",
		    "num_hidden_layers": "n_layer",
		}
		def __init__( self					,				SCREAMING_SNAKE_CASE__=5_04_00					,				SCREAMING_SNAKE_CASE__=20_48					,				SCREAMING_SNAKE_CASE__=40_96					,				SCREAMING_SNAKE_CASE__=28					,				SCREAMING_SNAKE_CASE__=16					,				SCREAMING_SNAKE_CASE__=64					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__="gelu_new"					,				SCREAMING_SNAKE_CASE__=0.0					,				SCREAMING_SNAKE_CASE__=0.0					,				SCREAMING_SNAKE_CASE__=0.0					,				SCREAMING_SNAKE_CASE__=1E-5					,				SCREAMING_SNAKE_CASE__=0.0_2					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=5_02_56					,				SCREAMING_SNAKE_CASE__=5_02_56					,				SCREAMING_SNAKE_CASE__=False					,				**SCREAMING_SNAKE_CASE__					,				)      ->	Dict:

					"""simple docstring"""



					_UpperCamelCase					:List[Any]					     =	vocab_size
					_UpperCamelCase					:Union[str, Any]					     =	n_positions
					_UpperCamelCase					:Dict					     =	n_embd
					_UpperCamelCase					:List[Any]					     =	n_layer
					_UpperCamelCase					:Optional[Any]					     =	n_head
					_UpperCamelCase					:str					     =	n_inner
					_UpperCamelCase					:str					     =	rotary_dim
					_UpperCamelCase					:Optional[Any]					     =	activation_function
					_UpperCamelCase					:Union[str, Any]					     =	resid_pdrop
					_UpperCamelCase					:List[str]					     =	embd_pdrop
					_UpperCamelCase					:Optional[int]					     =	attn_pdrop
					_UpperCamelCase					:Dict					     =	layer_norm_epsilon
					_UpperCamelCase					:int					     =	initializer_range
					_UpperCamelCase					:List[str]					     =	use_cache

					_UpperCamelCase					:Optional[int]					     =	bos_token_id
					_UpperCamelCase					:Any					     =	eos_token_id

					super().__init__(
					    bos_token_id=UpperCAmelCase__					,				eos_token_id=UpperCAmelCase__					,				tie_word_embeddings=UpperCAmelCase__					,				**UpperCAmelCase__   )






class      A(  UpperCAmelCase__		):

		"""simple docstring"""





		def __init__( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__ = "default"					,				SCREAMING_SNAKE_CASE__ = None					,				SCREAMING_SNAKE_CASE__ = False					,				)      ->	Optional[int]:

					"""simple docstring"""



					super().__init__(UpperCAmelCase__					,				task=UpperCAmelCase__					,				patching_specs=UpperCAmelCase__					,				use_past=UpperCAmelCase__   )
					if not getattr(self._config					,				'''pad_token_id'''					,				UpperCAmelCase__   ):
								# TODO: how to do that better?
								_UpperCamelCase					:Dict					     =	0
		@property
		def   _UpperCamelCase( self   )      ->	Mapping[str, Mapping[int, str]]:

					"""simple docstring"""



					_UpperCamelCase					:Any					     =	OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}   )
					if self.use_past:
								self.fill_with_past_key_values_(UpperCAmelCase__					,				direction='''inputs'''   )
								_UpperCamelCase					:Union[str, Any]					     =	{0: '''batch''', 1: '''past_sequence + sequence'''}
					else:
								_UpperCamelCase					:Any					     =	{0: '''batch''', 1: '''sequence'''}

					return common_inputs
		@property
		def   _UpperCamelCase( self   )      ->	int:

					"""simple docstring"""



					return self._config.n_layer
		@property
		def   _UpperCamelCase( self   )      ->	int:

					"""simple docstring"""



					return self._config.n_head
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__ = -1					,				SCREAMING_SNAKE_CASE__ = -1					,				SCREAMING_SNAKE_CASE__ = False					,				SCREAMING_SNAKE_CASE__ = None					,				)      ->	Mapping[str, Any]:

					"""simple docstring"""



					_UpperCamelCase					:List[Any]					     =	super(UpperCAmelCase__					,				self   ).generate_dummy_inputs(
					    UpperCAmelCase__					,				batch_size=UpperCAmelCase__					,				seq_length=UpperCAmelCase__					,				is_pair=UpperCAmelCase__					,				framework=UpperCAmelCase__   )

					# We need to order the input in the way they appears in the forward()
					_UpperCamelCase					:List[str]					     =	OrderedDict({'''input_ids''': common_inputs['''input_ids''']}   )

					# Need to add the past_keys
					if self.use_past:
								if not is_torch_available():
											raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''   )
								else:
											import torch

											_UpperCamelCase     ,      _UpperCamelCase					:List[str]					     =	common_inputs['''input_ids'''].shape
											# Not using the same length for past_key_values
											_UpperCamelCase					:str					     =	seqlen + 2
											_UpperCamelCase					:Optional[int]					     =	(
											    batch,
											    self.num_attention_heads,
											    past_key_values_length,
											    self._config.hidden_size // self.num_attention_heads,
											)
											_UpperCamelCase					:Any					     =	[
											    (torch.zeros(UpperCAmelCase__   ), torch.zeros(UpperCAmelCase__   )) for _ in range(self.num_layers   )
											]

					_UpperCamelCase					:Optional[Any]					     =	common_inputs['''attention_mask''']
					if self.use_past:
								_UpperCamelCase					:int					     =	ordered_inputs['''attention_mask'''].dtype
								_UpperCamelCase					:List[str]					     =	torch.cat(
								    [ordered_inputs['''attention_mask'''], torch.ones(UpperCAmelCase__					,				UpperCAmelCase__					,				dtype=UpperCAmelCase__   )]					,				dim=1   )

					return ordered_inputs





		@property
		def   _UpperCamelCase( self   )      ->	int:

					"""simple docstring"""



					return 13







# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


_lowerCamelCase      :						Optional[Any]							=  """Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""







def SCREAMING_SNAKE_CASE							(			)  ->			Dict:



       """simple docstring"""

       A__ 					=       _ask_options(
           '''In which compute environment are you running?'''					,							['''This machine''', '''AWS (Amazon SageMaker)''']					,							_convert_compute_environment					,							)
       if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
              A__ 					=       get_sagemaker_input()
       else:
              A__ 					=       get_cluster_input()
       return config







def SCREAMING_SNAKE_CASE							(			lowercase_=None		)  ->			List[Any]:



       """simple docstring"""

       if subparsers is not None:
              A__ 					=       subparsers.add_parser('''config'''					,							description=lowercase_		)
       else:
              A__ 					=       argparse.ArgumentParser('''Accelerate config command'''					,							description=lowercase_		)

       parser.add_argument(
           '''--config_file'''					,							default=lowercase_					,							help=(
               '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
               '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
               '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
               '''with \'huggingface\'.'''
           )					,							)

       if subparsers is not None:
              parser.set_defaults(func=lowercase_		)
       return parser







def SCREAMING_SNAKE_CASE							(			lowercase_		)  ->			Any:



       """simple docstring"""

       A__ 					=       get_user_input()
       if args.config_file is not None:
              A__ 					=       args.config_file
       else:
              if not os.path.isdir(lowercase_		):
                     os.makedirs(lowercase_		)
              A__ 					=       default_yaml_config_file

       if config_file.endswith('''.json'''		):
              config.to_json_file(lowercase_		)
       else:
              config.to_yaml_file(lowercase_		)
       print(f"""accelerate configuration saved at {config_file}"""		)







def SCREAMING_SNAKE_CASE							(			)  ->			Union[str, Any]:



       """simple docstring"""

       A__ 					=       config_command_parser()
       A__ 					=       parser.parse_args()
       config_command(lowercase_		)


if __name__ == "__main__":
  main()


