






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
       import botoa  # noqa: F401


def 				__lowerCamelCase     (							__lowerCAmelCase							:					int							)     ->  Optional[int]:
   __UpperCamelCase	:			str					=					botoa.client("""iam"""							)

   __UpperCamelCase	:			Optional[Any]					=					{
       """Version""": """2012-10-17""",
       """Statement""": [
           {"""Effect""": """Allow""", """Principal""": {"""Service""": """sagemaker.amazonaws.com"""}, """Action""": """sts:AssumeRole"""}
       ],
   }
   try:
      # create the role, associated with the chosen trust policy
      iam_client.create_role(
          RoleName=__lowerCAmelCase					,							AssumeRolePolicyDocument=json.dumps(__lowerCAmelCase					,							indent=2							)							)
      __UpperCamelCase	:			Any					=					{
          """Version""": """2012-10-17""",
          """Statement""": [
              {
                  """Effect""": """Allow""",
                  """Action""": [
                      """sagemaker:*""",
                      """ecr:GetDownloadUrlForLayer""",
                      """ecr:BatchGetImage""",
                      """ecr:BatchCheckLayerAvailability""",
                      """ecr:GetAuthorizationToken""",
                      """cloudwatch:PutMetricData""",
                      """cloudwatch:GetMetricData""",
                      """cloudwatch:GetMetricStatistics""",
                      """cloudwatch:ListMetrics""",
                      """logs:CreateLogGroup""",
                      """logs:CreateLogStream""",
                      """logs:DescribeLogStreams""",
                      """logs:PutLogEvents""",
                      """logs:GetLogEvents""",
                      """s3:CreateBucket""",
                      """s3:ListBucket""",
                      """s3:GetBucketLocation""",
                      """s3:GetObject""",
                      """s3:PutObject""",
                  ],
                  """Resource""": """*""",
              }
          ],
      }
      # attach policy to role
      iam_client.put_role_policy(
          RoleName=__lowerCAmelCase					,							PolicyName=f'{role_name}_policy_permission'					,							PolicyDocument=json.dumps(__lowerCAmelCase					,							indent=2							)					,							)
   except iam_client.exceptions.EntityAlreadyExistsException:
      print(f'role {role_name} already exists. Using existing one'							)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					List[str]							)     ->  int:
   __UpperCamelCase	:			str					=					botoa.client("""iam"""							)
   return iam_client.get_role(RoleName=__lowerCAmelCase							)["Role"]["Arn"]



def 				__lowerCamelCase     (							)     ->  Any:
   __UpperCamelCase	:			Dict					=					_ask_options(
       """How do you want to authorize?"""					,							["""AWS Profile""", """Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) """]					,							__lowerCAmelCase					,							)
   __UpperCamelCase	:			str					=					None
   if credentials_configuration == 0:
      __UpperCamelCase	:			str					=					_ask_field("""Enter your AWS Profile name: [default] """					,							default="""default"""							)
      __UpperCamelCase	:			Any					=					aws_profile
   else:
      print(
          """Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"""
          """`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"""							)
      __UpperCamelCase	:			Optional[Any]					=					_ask_field("""AWS Access Key ID: """							)
      __UpperCamelCase	:			int					=					aws_access_key_id

      __UpperCamelCase	:			str					=					_ask_field("""AWS Secret Access Key: """							)
      __UpperCamelCase	:			Optional[int]					=					aws_secret_access_key

   __UpperCamelCase	:			Any					=					_ask_field("""Enter your AWS Region: [us-east-1]"""					,							default="""us-east-1"""							)
   __UpperCamelCase	:			int					=					aws_region

   __UpperCamelCase	:			List[Any]					=					_ask_options(
       """Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"""					,							["""Provide IAM Role name""", """Create new IAM role using credentials"""]					,							__lowerCAmelCase					,							)
   if role_management == 0:
      __UpperCamelCase	:			List[Any]					=					_ask_field("""Enter your IAM role name: """							)
   else:
      __UpperCamelCase	:			Union[str, Any]					=					"""accelerate_sagemaker_execution_role"""
      print(f'Accelerate will create an iam role "{iam_role_name}" using the provided credentials'							)
      _create_iam_role_for_sagemaker(__lowerCAmelCase							)

   __UpperCamelCase	:			Any					=					_ask_field(
       """Do you want to use custom Docker image? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
   __UpperCamelCase	:			List[Any]					=					None
   if is_custom_docker_image:
      __UpperCamelCase	:			List[str]					=					_ask_field("""Enter your Docker image: """					,							lambda __lowerCAmelCase							: str(__lowerCAmelCase							).lower()							)

   __UpperCamelCase	:			int					=					_ask_field(
       """Do you want to provide SageMaker input channels with data locations? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
   __UpperCamelCase	:			List[str]					=					None
   if is_sagemaker_inputs_enabled:
      __UpperCamelCase	:			str					=					_ask_field(
          """Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): """					,							lambda __lowerCAmelCase							: str(__lowerCAmelCase							).lower()					,							)

   __UpperCamelCase	:			Any					=					_ask_field(
       """Do you want to enable SageMaker metrics? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
   __UpperCamelCase	:			Optional[Any]					=					None
   if is_sagemaker_metrics_enabled:
      __UpperCamelCase	:			List[str]					=					_ask_field(
          """Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): """					,							lambda __lowerCAmelCase							: str(__lowerCAmelCase							).lower()					,							)

   __UpperCamelCase	:			Dict					=					_ask_options(
       """What is the distributed mode?"""					,							["""No distributed training""", """Data parallelism"""]					,							_convert_sagemaker_distributed_mode					,							)
   __UpperCamelCase	:			List[Any]					=					{}
   __UpperCamelCase	:			Dict					=					_ask_field(
       """Do you wish to optimize your script with torch dynamo?[yes/NO]:"""					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
   if use_dynamo:
      __UpperCamelCase	:			Tuple					=					"""dynamo_"""
      __UpperCamelCase	:			Any					=					_ask_options(
          """Which dynamo backend would you like to use?"""					,							[x.lower() for x in DYNAMO_BACKENDS]					,							_convert_dynamo_backend					,							default=2					,							)
      __UpperCamelCase	:			List[Any]					=					_ask_field(
          """Do you want to customize the defaults sent to torch.compile? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)

      if use_custom_options:
         __UpperCamelCase	:			Tuple					=					_ask_options(
             """Which mode do you want to use?"""					,							__lowerCAmelCase					,							lambda __lowerCAmelCase							: TORCH_DYNAMO_MODES[int(__lowerCAmelCase							)]					,							default="""default"""					,							)
         __UpperCamelCase	:			Optional[Any]					=					_ask_field(
             """Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
         __UpperCamelCase	:			Union[str, Any]					=					_ask_field(
             """Do you want to enable dynamic shape tracing? [yes/NO]: """					,							_convert_yes_no_to_bool					,							default=__lowerCAmelCase					,							error_message="""Please enter yes or no."""					,							)
   __UpperCamelCase	:			Union[str, Any]					=					"""Which EC2 instance type you want to use for your training?"""
   if distributed_type != SageMakerDistributedType.NO:
      __UpperCamelCase	:			Tuple					=					_ask_options(
          __lowerCAmelCase					,							__lowerCAmelCase					,							lambda __lowerCAmelCase							: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(__lowerCAmelCase							)]							)
   else:
      eca_instance_query += "? [ml.p3.2xlarge]:"
      __UpperCamelCase	:			str					=					_ask_field(__lowerCAmelCase					,							lambda __lowerCAmelCase							: str(__lowerCAmelCase							).lower()					,							default="""ml.p3.2xlarge"""							)

   __UpperCamelCase	:			str					=					1
   if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
      __UpperCamelCase	:			int					=					_ask_field(
          """How many machines do you want use? [1]: """					,							__lowerCAmelCase					,							default=1					,							)

   __UpperCamelCase	:			Optional[Any]					=					_ask_options(
       """Do you wish to use FP16 or BF16 (mixed precision)?"""					,							["""no""", """fp16""", """bf16""", """fp8"""]					,							_convert_mixed_precision					,							)

   if use_dynamo and mixed_precision == "no":
      print(
          """Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."""							)

   return SageMakerConfig(
       image_uri=__lowerCAmelCase					,							compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER					,							distributed_type=__lowerCAmelCase					,							use_cpu=__lowerCAmelCase					,							dynamo_config=__lowerCAmelCase					,							eca_instance_type=__lowerCAmelCase					,							profile=__lowerCAmelCase					,							region=__lowerCAmelCase					,							iam_role_name=__lowerCAmelCase					,							mixed_precision=__lowerCAmelCase					,							num_machines=__lowerCAmelCase					,							sagemaker_inputs_file=__lowerCAmelCase					,							sagemaker_metrics_file=__lowerCAmelCase					,							)











import os
from collections import namedtuple

import pytest

from datasets import ClassLabel, Features, Sequence, Value
from datasets.commands.test import TestCommand
from datasets.info import DatasetInfo, DatasetInfosDict


UpperCamelCase    							=      namedtuple(
    '_TestCommandArgs',
    [
        'dataset',
        'name',
        'cache_dir',
        'data_dir',
        'all_configs',
        'save_infos',
        'ignore_verifications',
        'force_redownload',
        'clear_cache',
    ],
    defaults=[None, None, None, False, False, False, False, False],
)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					List[Any]					,							__lowerCAmelCase							:					Any							)     ->  Optional[int]:
   return (abs(source - target							) / target) < 0.01



@pytest.mark.integration
def 				__lowerCamelCase     (							__lowerCAmelCase							:					str							)     ->  Union[str, Any]:
   __UpperCamelCase	:			Dict					=					_TestCommandArgs(dataset=__lowerCAmelCase					,							all_configs=__lowerCAmelCase					,							save_infos=__lowerCAmelCase							)
   __UpperCamelCase	:			List[Any]					=					TestCommand(*__lowerCAmelCase							)
   test_command.run()
   __UpperCamelCase	:			Any					=					os.path.join(__lowerCAmelCase					,							"""README.md"""							)
   assert os.path.exists(__lowerCAmelCase							)
   __UpperCamelCase	:			Optional[int]					=					DatasetInfosDict.from_directory(__lowerCAmelCase							)
   __UpperCamelCase	:			List[Any]					=					DatasetInfosDict(
       {
           """default""": DatasetInfo(
               features=Features(
                   {
                       """tokens""": Sequence(Value("""string"""							)							),
                       """ner_tags""": Sequence(
                           ClassLabel(names=["""O""", """B-PER""", """I-PER""", """B-ORG""", """I-ORG""", """B-LOC""", """I-LOC"""]							)							),
                       """langs""": Sequence(Value("""string"""							)							),
                       """spans""": Sequence(Value("""string"""							)							),
                   }							)					,							splits=[
                   {
                       """name""": """train""",
                       """num_bytes""": 2351563,
                       """num_examples""": 10000,
                   },
                   {
                       """name""": """validation""",
                       """num_bytes""": 238418,
                       """num_examples""": 1000,
                   },
               ]					,							download_size=3940680					,							dataset_size=2589981					,							)
       }							)
   assert dataset_infos.keys() == expected_dataset_infos.keys()
   for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
      __UpperCamelCase	,     __UpperCamelCase	:			Union[str, Any]					=					getattr(dataset_infos["""default"""]					,							__lowerCAmelCase							), getattr(expected_dataset_infos["""default"""]					,							__lowerCAmelCase							)
      if key == "num_bytes":
         assert is_apercent_close(__lowerCAmelCase					,							__lowerCAmelCase							)
      elif key == "splits":
         assert list(__lowerCAmelCase							) == list(__lowerCAmelCase							)
         for split in result:
            assert result[split].name == expected[split].name
            assert result[split].num_examples == expected[split].num_examples
            assert is_apercent_close(result[split].num_bytes					,							expected[split].num_bytes							)
      else:
         result == expected



