






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
   import botoa  # noqa: F401

def   snake_case (UpperCAmelCase__  )	->    Union[str, Any]:
  UpperCamelCase_: Union[str, Any]   				=    botoa.client('iam'  )

  UpperCamelCase_: Dict   				=    {
      'Version': '2012-10-17',
      'Statement': [
          {'Effect': 'Allow', 'Principal': {'Service': 'sagemaker.amazonaws.com'}, 'Action': 'sts:AssumeRole'}
      ],
  }
  try:
    # create the role, associated with the chosen trust policy
    iam_client.create_role(
        RoleName=UpperCAmelCase__     ,   AssumeRolePolicyDocument=json.dumps(UpperCAmelCase__     ,   indent=2  )  )
    UpperCamelCase_: str   				=    {
        'Version': '2012-10-17',
        'Statement': [
            {
                'Effect': 'Allow',
                'Action': [
                    'sagemaker:*',
                    'ecr:GetDownloadUrlForLayer',
                    'ecr:BatchGetImage',
                    'ecr:BatchCheckLayerAvailability',
                    'ecr:GetAuthorizationToken',
                    'cloudwatch:PutMetricData',
                    'cloudwatch:GetMetricData',
                    'cloudwatch:GetMetricStatistics',
                    'cloudwatch:ListMetrics',
                    'logs:CreateLogGroup',
                    'logs:CreateLogStream',
                    'logs:DescribeLogStreams',
                    'logs:PutLogEvents',
                    'logs:GetLogEvents',
                    's3:CreateBucket',
                    's3:ListBucket',
                    's3:GetBucketLocation',
                    's3:GetObject',
                    's3:PutObject',
                ],
                'Resource': '*',
            }
        ],
    }
    # attach policy to role
    iam_client.put_role_policy(
        RoleName=UpperCAmelCase__     ,   PolicyName=F'''{role_name}_policy_permission'''     ,   PolicyDocument=json.dumps(UpperCAmelCase__     ,   indent=2  )     ,   )
  except iam_client.exceptions.EntityAlreadyExistsException:
    print(F'''role {role_name} already exists. Using existing one'''  )

def   snake_case (UpperCAmelCase__  )	->    Tuple:
  UpperCamelCase_: Optional[Any]   				=    botoa.client('iam'  )
  return iam_client.get_role(RoleName=UpperCAmelCase__  )["Role"]["Arn"]

def   snake_case ()	->    str:
  UpperCamelCase_: Tuple   				=    _ask_options(
      'How do you want to authorize?'     ,   ['AWS Profile', 'Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) ']     ,   UpperCAmelCase__     ,   )
  UpperCamelCase_: int   				=    None
  if credentials_configuration == 0:
    UpperCamelCase_: List[str]   				=    _ask_field('Enter your AWS Profile name: [default] '     ,   default='default'  )
    UpperCamelCase_: str   				=    aws_profile
  else:
    print(
        'Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'
        '`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`'  )
    UpperCamelCase_: Union[str, Any]   				=    _ask_field('AWS Access Key ID: '  )
    UpperCamelCase_: List[str]   				=    aws_access_key_id

    UpperCamelCase_: str   				=    _ask_field('AWS Secret Access Key: '  )
    UpperCamelCase_: List[str]   				=    aws_secret_access_key

  UpperCamelCase_: Tuple   				=    _ask_field('Enter your AWS Region: [us-east-1]'     ,   default='us-east-1'  )
  UpperCamelCase_: Any   				=    aws_region

  UpperCamelCase_: str   				=    _ask_options(
      'Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?'     ,   ['Provide IAM Role name', 'Create new IAM role using credentials']     ,   UpperCAmelCase__     ,   )
  if role_management == 0:
    UpperCamelCase_: Dict   				=    _ask_field('Enter your IAM role name: '  )
  else:
    UpperCamelCase_: Dict   				=    'accelerate_sagemaker_execution_role'
    print(F'''Accelerate will create an iam role "{iam_role_name}" using the provided credentials'''  )
    _create_iam_role_for_sagemaker(UpperCAmelCase__  )

  UpperCamelCase_: List[str]   				=    _ask_field(
      'Do you want to use custom Docker image? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
  UpperCamelCase_: Optional[int]   				=    None
  if is_custom_docker_image:
    UpperCamelCase_: List[str]   				=    _ask_field('Enter your Docker image: '     ,   lambda UpperCAmelCase__  : str(UpperCAmelCase__  ).lower()  )

  UpperCamelCase_: Dict   				=    _ask_field(
      'Do you want to provide SageMaker input channels with data locations? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
  UpperCamelCase_: Any   				=    None
  if is_sagemaker_inputs_enabled:
    UpperCamelCase_: Optional[Any]   				=    _ask_field(
        'Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): '     ,   lambda UpperCAmelCase__  : str(UpperCAmelCase__  ).lower()     ,   )

  UpperCamelCase_: Optional[int]   				=    _ask_field(
      'Do you want to enable SageMaker metrics? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
  UpperCamelCase_: str   				=    None
  if is_sagemaker_metrics_enabled:
    UpperCamelCase_: Dict   				=    _ask_field(
        'Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): '     ,   lambda UpperCAmelCase__  : str(UpperCAmelCase__  ).lower()     ,   )

  UpperCamelCase_: Optional[int]   				=    _ask_options(
      'What is the distributed mode?'     ,   ['No distributed training', 'Data parallelism']     ,   _convert_sagemaker_distributed_mode     ,   )
  UpperCamelCase_: str   				=    {}
  UpperCamelCase_: Tuple   				=    _ask_field(
      'Do you wish to optimize your script with torch dynamo?[yes/NO]:'     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
  if use_dynamo:
    UpperCamelCase_: Optional[int]   				=    'dynamo_'
    UpperCamelCase_: int   				=    _ask_options(
        'Which dynamo backend would you like to use?'     ,   [x.lower() for x in DYNAMO_BACKENDS]     ,   _convert_dynamo_backend     ,   default=2     ,   )
    UpperCamelCase_: Optional[int]   				=    _ask_field(
        'Do you want to customize the defaults sent to torch.compile? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )

    if use_custom_options:
      UpperCamelCase_: Optional[Any]   				=    _ask_options(
          'Which mode do you want to use?'     ,   UpperCAmelCase__     ,   lambda UpperCAmelCase__  : TORCH_DYNAMO_MODES[int(UpperCAmelCase__  )]     ,   default='default'     ,   )
      UpperCamelCase_: Optional[int]   				=    _ask_field(
          'Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
      UpperCamelCase_: Tuple   				=    _ask_field(
          'Do you want to enable dynamic shape tracing? [yes/NO]: '     ,   _convert_yes_no_to_bool     ,   default=UpperCAmelCase__     ,   error_message='Please enter yes or no.'     ,   )
  UpperCamelCase_: int   				=    'Which EC2 instance type you want to use for your training?'
  if distributed_type != SageMakerDistributedType.NO:
    UpperCamelCase_: List[str]   				=    _ask_options(
        UpperCAmelCase__     ,   UpperCAmelCase__     ,   lambda UpperCAmelCase__  : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(UpperCAmelCase__  )]  )
  else:
    eca_instance_query += "? [ml.p3.2xlarge]:"
    UpperCamelCase_: Optional[int]   				=    _ask_field(UpperCAmelCase__     ,   lambda UpperCAmelCase__  : str(UpperCAmelCase__  ).lower()     ,   default='ml.p3.2xlarge'  )

  UpperCamelCase_: Dict   				=    1
  if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
    UpperCamelCase_: str   				=    _ask_field(
        'How many machines do you want use? [1]: '     ,   UpperCAmelCase__     ,   default=1     ,   )

  UpperCamelCase_: Union[str, Any]   				=    _ask_options(
      'Do you wish to use FP16 or BF16 (mixed precision)?'     ,   ['no', 'fp16', 'bf16', 'fp8']     ,   _convert_mixed_precision     ,   )

  if use_dynamo and mixed_precision == "no":
    print(
        'Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.'  )

  return SageMakerConfig(
      image_uri=UpperCAmelCase__     ,   compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER     ,   distributed_type=UpperCAmelCase__     ,   use_cpu=UpperCAmelCase__     ,   dynamo_config=UpperCAmelCase__     ,   eca_instance_type=UpperCAmelCase__     ,   profile=UpperCAmelCase__     ,   region=UpperCAmelCase__     ,   iam_role_name=UpperCAmelCase__     ,   mixed_precision=UpperCAmelCase__     ,   num_machines=UpperCAmelCase__     ,   sagemaker_inputs_file=UpperCAmelCase__     ,   sagemaker_metrics_file=UpperCAmelCase__     ,   )







from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:		Tuple		     =			logging.get_logger(__name__)

A_		:		Dict		     =			{
    'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/config.json',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       a    :		Tuple									='''xglm'''
       a    :		List[Any]									=['''past_key_values''']

       a    :		Union[str, Any]									={
           '''num_attention_heads''': '''attention_heads''',
           '''hidden_size''': '''d_model''',
           '''num_hidden_layers''': '''num_layers''',
       }





       def __init__( self  ,      _lowerCamelCase=2_5_6_0_0_8  ,      _lowerCamelCase=2_0_4_8  ,      _lowerCamelCase=1_0_2_4  ,      _lowerCamelCase=4_0_9_6  ,      _lowerCamelCase=2_4  ,      _lowerCamelCase=1_6  ,      _lowerCamelCase="gelu"  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.0  ,      _lowerCamelCase=0.0  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=2  ,      _lowerCamelCase=1  ,      _lowerCamelCase=0  ,      _lowerCamelCase=2  ,      **_lowerCamelCase  ,      ):
         UpperCamelCase_: Optional[Any]   				=    vocab_size
         UpperCamelCase_: Optional[int]   				=    max_position_embeddings
         UpperCamelCase_: List[str]   				=    d_model
         UpperCamelCase_: List[Any]   				=    ffn_dim
         UpperCamelCase_: List[Any]   				=    num_layers
         UpperCamelCase_: List[Any]   				=    attention_heads
         UpperCamelCase_: Tuple   				=    activation_function
         UpperCamelCase_: Tuple   				=    dropout
         UpperCamelCase_: Tuple   				=    attention_dropout
         UpperCamelCase_: Optional[Any]   				=    activation_dropout
         UpperCamelCase_: List[str]   				=    layerdrop
         UpperCamelCase_: Any   				=    init_std
         UpperCamelCase_: Any   				=    scale_embedding  # scale factor will be sqrt(d_model) if True
         UpperCamelCase_: Union[str, Any]   				=    use_cache

         super().__init__(
             pad_token_id=_lowerCamelCase  ,      bos_token_id=_lowerCamelCase  ,      eos_token_id=_lowerCamelCase  ,      decoder_start_token_id=_lowerCamelCase  ,      **_lowerCamelCase  ,      )