





# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
     import botoa  # noqa: F401




def     a__ (					lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=botoa.client("iam"  )

   UpperCAmelCase_											={
       "Version": "2012-10-17",
       "Statement": [
           {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
       ],
   }
   try:
      # create the role, associated with the chosen trust policy
      iam_client.create_role(
          RoleName=lowercase__  ,    AssumeRolePolicyDocument=json.dumps(lowercase__  ,    indent=2  )  )
      UpperCAmelCase_											={
          "Version": "2012-10-17",
          "Statement": [
              {
                  "Effect": "Allow",
                  "Action": [
                      "sagemaker:*",
                      "ecr:GetDownloadUrlForLayer",
                      "ecr:BatchGetImage",
                      "ecr:BatchCheckLayerAvailability",
                      "ecr:GetAuthorizationToken",
                      "cloudwatch:PutMetricData",
                      "cloudwatch:GetMetricData",
                      "cloudwatch:GetMetricStatistics",
                      "cloudwatch:ListMetrics",
                      "logs:CreateLogGroup",
                      "logs:CreateLogStream",
                      "logs:DescribeLogStreams",
                      "logs:PutLogEvents",
                      "logs:GetLogEvents",
                      "s3:CreateBucket",
                      "s3:ListBucket",
                      "s3:GetBucketLocation",
                      "s3:GetObject",
                      "s3:PutObject",
                  ],
                  "Resource": "*",
              }
          ],
      }
      # attach policy to role
      iam_client.put_role_policy(
          RoleName=lowercase__  ,    PolicyName=F'{role_name}_policy_permission'  ,    PolicyDocument=json.dumps(lowercase__  ,    indent=2  )  ,    )
   except iam_client.exceptions.EntityAlreadyExistsException:
      print(F'role {role_name} already exists. Using existing one'  )




def     a__ (					lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=botoa.client("iam"  )
   return iam_client.get_role(RoleName=lowercase__  )["Role"]["Arn"]





def     a__ (					):






   '''simple docstring'''






   UpperCAmelCase_											=_ask_options(
       "How do you want to authorize?"  ,    ["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "]  ,    lowercase__  ,    )
   UpperCAmelCase_											=None
   if credentials_configuration == 0:
      UpperCAmelCase_											=_ask_field("Enter your AWS Profile name: [default] "  ,    default="default"  )
      UpperCAmelCase_											=aws_profile
   else:
      print(
          "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"
          "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"  )
      UpperCAmelCase_											=_ask_field("AWS Access Key ID: "  )
      UpperCAmelCase_											=aws_access_key_id

      UpperCAmelCase_											=_ask_field("AWS Secret Access Key: "  )
      UpperCAmelCase_											=aws_secret_access_key

   UpperCAmelCase_											=_ask_field("Enter your AWS Region: [us-east-1]"  ,    default="us-east-1"  )
   UpperCAmelCase_											=aws_region

   UpperCAmelCase_											=_ask_options(
       "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"  ,    ["Provide IAM Role name", "Create new IAM role using credentials"]  ,    lowercase__  ,    )
   if role_management == 0:
      UpperCAmelCase_											=_ask_field("Enter your IAM role name: "  )
   else:
      UpperCAmelCase_											="accelerate_sagemaker_execution_role"
      print(F'Accelerate will create an iam role "{iam_role_name}" using the provided credentials'  )
      _create_iam_role_for_sagemaker(lowercase__  )

   UpperCAmelCase_											=_ask_field(
       "Do you want to use custom Docker image? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
   UpperCAmelCase_											=None
   if is_custom_docker_image:
      UpperCAmelCase_											=_ask_field("Enter your Docker image: "  ,    lambda lowercase__  : str(lowercase__  ).lower()  )

   UpperCAmelCase_											=_ask_field(
       "Do you want to provide SageMaker input channels with data locations? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
   UpperCAmelCase_											=None
   if is_sagemaker_inputs_enabled:
      UpperCAmelCase_											=_ask_field(
          "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): "  ,    lambda lowercase__  : str(lowercase__  ).lower()  ,    )

   UpperCAmelCase_											=_ask_field(
       "Do you want to enable SageMaker metrics? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
   UpperCAmelCase_											=None
   if is_sagemaker_metrics_enabled:
      UpperCAmelCase_											=_ask_field(
          "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): "  ,    lambda lowercase__  : str(lowercase__  ).lower()  ,    )

   UpperCAmelCase_											=_ask_options(
       "What is the distributed mode?"  ,    ["No distributed training", "Data parallelism"]  ,    _convert_sagemaker_distributed_mode  ,    )
   UpperCAmelCase_											={}
   UpperCAmelCase_											=_ask_field(
       "Do you wish to optimize your script with torch dynamo?[yes/NO]:"  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
   if use_dynamo:
      UpperCAmelCase_											="dynamo_"
      UpperCAmelCase_											=_ask_options(
          "Which dynamo backend would you like to use?"  ,    [x.lower() for x in DYNAMO_BACKENDS]  ,    _convert_dynamo_backend  ,    default=2  ,    )
      UpperCAmelCase_											=_ask_field(
          "Do you want to customize the defaults sent to torch.compile? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )

      if use_custom_options:
         UpperCAmelCase_											=_ask_options(
             "Which mode do you want to use?"  ,    lowercase__  ,    lambda lowercase__  : TORCH_DYNAMO_MODES[int(lowercase__  )]  ,    default="default"  ,    )
         UpperCAmelCase_											=_ask_field(
             "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
         UpperCAmelCase_											=_ask_field(
             "Do you want to enable dynamic shape tracing? [yes/NO]: "  ,    _convert_yes_no_to_bool  ,    default=lowercase__  ,    error_message="Please enter yes or no."  ,    )
   UpperCAmelCase_											="Which EC2 instance type you want to use for your training?"
   if distributed_type != SageMakerDistributedType.NO:
      UpperCAmelCase_											=_ask_options(
          lowercase__  ,    lowercase__  ,    lambda lowercase__  : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(lowercase__  )]  )
   else:
      eca_instance_query += "? [ml.p3.2xlarge]:"
      UpperCAmelCase_											=_ask_field(lowercase__  ,    lambda lowercase__  : str(lowercase__  ).lower()  ,    default="ml.p3.2xlarge"  )

   UpperCAmelCase_											=1
   if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
      UpperCAmelCase_											=_ask_field(
          "How many machines do you want use? [1]: "  ,    lowercase__  ,    default=1  ,    )

   UpperCAmelCase_											=_ask_options(
       "Do you wish to use FP16 or BF16 (mixed precision)?"  ,    ["no", "fp16", "bf16", "fp8"]  ,    _convert_mixed_precision  ,    )

   if use_dynamo and mixed_precision == "no":
      print(
          "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."  )

   return SageMakerConfig(
       image_uri=lowercase__  ,    compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER  ,    distributed_type=lowercase__  ,    use_cpu=lowercase__  ,    dynamo_config=lowercase__  ,    eca_instance_type=lowercase__  ,    profile=lowercase__  ,    region=lowercase__  ,    iam_role_name=lowercase__  ,    mixed_precision=lowercase__  ,    num_machines=lowercase__  ,    sagemaker_inputs_file=lowercase__  ,    sagemaker_metrics_file=lowercase__  ,    )











from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml

class      A       (	yaml.SafeLoader						):



      def 	lowerCAmelCase__    (	self:						Optional[Any]		,  _lowerCAmelCase:						int       )						->    Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_											=[self.constructed_objects[key_node] for key_node, _ in node.value]
         UpperCAmelCase_											=[tuple(_lowerCAmelCase       ) if isinstance(_lowerCAmelCase		,  _lowerCAmelCase       ) else key for key in keys]
         UpperCAmelCase_											=Counter(_lowerCAmelCase       )
         UpperCAmelCase_											=[key for key in counter if counter[key] > 1]
         if duplicate_keys:
            raise TypeError(F'Got duplicate yaml keys: {duplicate_keys}'       )






      def 	lowerCAmelCase__    (	self:						Dict		,  _lowerCAmelCase:						List[str]		,  _lowerCAmelCase:						Optional[Any]=False       )						->    int:
         '''simple docstring'''

         UpperCAmelCase_											=super().construct_mapping(_lowerCAmelCase		,  deep=_lowerCAmelCase       )
         self._check_no_duplicates_on_constructed_node(_lowerCAmelCase       )
         return mapping




def     a__ (					lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=list(readme_content.splitlines()  )
   if full_content and full_content[0] == "---" and "---" in full_content[1:]:
      UpperCAmelCase_											=full_content[1:].index("---"  ) + 1
      UpperCAmelCase_											="\n".join(full_content[1:sep_idx]  )
      return yamlblock, "\n".join(full_content[sep_idx + 1 :]  )

   return None, "\n".join(lowercase__  )

class      A       (	__lowercase						):
      # class attributes
      _snake_case							     ={'''train_eval_index'''}  # train-eval-index in the YAML metadata



      @classmethod
      def 	lowerCAmelCase__    (	cls:						Any		,  _lowerCAmelCase:						Path       )						->    "DatasetMetadata":
         '''simple docstring'''

         with open(_lowerCAmelCase		,  encoding="utf-8"       ) as readme_file:
            UpperCAmelCase_						,       UpperCAmelCase_											=_split_yaml_from_readme(readme_file.read()       )
         if yaml_string is not None:
            return cls.from_yaml_string(_lowerCAmelCase       )
         else:
            return cls()



      def 	lowerCAmelCase__    (	self:						Dict		,  _lowerCAmelCase:						Path       )						->    List[str]:
         '''simple docstring'''

         if path.exists():
            with open(_lowerCAmelCase		,  encoding="utf-8"       ) as readme_file:
               UpperCAmelCase_											=readme_file.read()
         else:
            UpperCAmelCase_											=None
         UpperCAmelCase_											=self._to_readme(_lowerCAmelCase       )
         with open(_lowerCAmelCase		,  "w"		,  encoding="utf-8"       ) as readme_file:
            readme_file.write(_lowerCAmelCase       )



      def 	lowerCAmelCase__    (	self:						str		,  _lowerCAmelCase:						Optional[str] = None       )						->    str:
         '''simple docstring'''

         if readme_content is not None:
            UpperCAmelCase_						,       UpperCAmelCase_											=_split_yaml_from_readme(_lowerCAmelCase       )
            UpperCAmelCase_											="---\n" + self.to_yaml_string() + "---\n" + content
         else:
            UpperCAmelCase_											="---\n" + self.to_yaml_string() + "---\n"
         return full_content



      @classmethod
      def 	lowerCAmelCase__    (	cls:						Optional[Any]		,  _lowerCAmelCase:						str       )						->    "DatasetMetadata":
         '''simple docstring'''

         UpperCAmelCase_											=yaml.load(_lowerCAmelCase		,  Loader=_NoDuplicateSafeLoader       ) or {}

         # Convert the YAML keys to DatasetMetadata fields
         UpperCAmelCase_											={
             (key.replace("-"		,  "_"       ) if key.replace("-"		,  "_"       ) in cls._FIELDS_WITH_DASHES else key): value
             for key, value in metadata_dict.items()
         }
         return cls(**_lowerCAmelCase       )






      def 	lowerCAmelCase__    (	self:						int       )						->    str:
         '''simple docstring'''

         return yaml.safe_dump(
             {
                 (key.replace("_"		,  "-"       ) if key in self._FIELDS_WITH_DASHES else key): value
                 for key, value in self.items()
             }		,  sort_keys=_lowerCAmelCase		,  allow_unicode=_lowerCAmelCase		,  encoding="utf-8"		,  ).decode("utf-8"       )

__lowercase  :   int						      ={
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
     from argparse import ArgumentParser

     __lowercase  :   List[Any]						      =ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
     ap.add_argument("""readme_filepath""")
     __lowercase  :   int						      =ap.parse_args()

     __lowercase  :   Optional[int]						      =Path(args.readme_filepath)
     __lowercase  :   Tuple						      =DatasetMetadata.from_readme(readme_filepath)
     print(dataset_metadata)
     dataset_metadata.to_readme(readme_filepath)




