

"""simple docstring"""




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
						import botoa  # noqa: F401

def 						A							(  snake_case__     ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   botoa.client("""iam"""     )

			SCREAMING_SNAKE_CASE__										=   {
			    """Version""": """2012-10-17""",
			    """Statement""": [
			        {"""Effect""": """Allow""", """Principal""": {"""Service""": """sagemaker.amazonaws.com"""}, """Action""": """sts:AssumeRole"""}
			    ],
			}
			try:
						# create the role, associated with the chosen trust policy
						iam_client.create_role(
						    RoleName=snake_case__		,					AssumeRolePolicyDocument=json.dumps(snake_case__		,					indent=2     )     )
						SCREAMING_SNAKE_CASE__										=   {
						    """Version""": """2012-10-17""",
						    """Statement""": [
						        {
						            """Effect""": """Allow""",
						            """Action""": [
						                """sagemaker:*""",
						                """ecr:GetDownloadUrlForLayer""",
						                """ecr:BatchGetImage""",
						                """ecr:BatchCheckLayerAvailability""",
						                """ecr:GetAuthorizationToken""",
						                """cloudwatch:PutMetricData""",
						                """cloudwatch:GetMetricData""",
						                """cloudwatch:GetMetricStatistics""",
						                """cloudwatch:ListMetrics""",
						                """logs:CreateLogGroup""",
						                """logs:CreateLogStream""",
						                """logs:DescribeLogStreams""",
						                """logs:PutLogEvents""",
						                """logs:GetLogEvents""",
						                """s3:CreateBucket""",
						                """s3:ListBucket""",
						                """s3:GetBucketLocation""",
						                """s3:GetObject""",
						                """s3:PutObject""",
						            ],
						            """Resource""": """*""",
						        }
						    ],
						}
						# attach policy to role
						iam_client.put_role_policy(
						    RoleName=snake_case__		,					PolicyName=f"""{role_name}_policy_permission"""		,					PolicyDocument=json.dumps(snake_case__		,					indent=2     )		,					)
			except iam_client.exceptions.EntityAlreadyExistsException:
						print(f"""role {role_name} already exists. Using existing one"""     )

def 						A							(  snake_case__     ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   botoa.client("""iam"""     )
			return iam_client.get_role(RoleName=snake_case__     )["Role"]["Arn"]




def 						A							(  ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   _ask_options(
			    """How do you want to authorize?"""		,					["""AWS Profile""", """Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) """]		,					snake_case__		,					)
			SCREAMING_SNAKE_CASE__										=   None
			if credentials_configuration == 0:
						SCREAMING_SNAKE_CASE__										=   _ask_field("""Enter your AWS Profile name: [default] """		,					default="""default"""     )
						SCREAMING_SNAKE_CASE__										=   aws_profile
			else:
						print(
						    """Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"""
						    """`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"""     )
						SCREAMING_SNAKE_CASE__										=   _ask_field("""AWS Access Key ID: """     )
						SCREAMING_SNAKE_CASE__										=   aws_access_key_id

						SCREAMING_SNAKE_CASE__										=   _ask_field("""AWS Secret Access Key: """     )
						SCREAMING_SNAKE_CASE__										=   aws_secret_access_key

			SCREAMING_SNAKE_CASE__										=   _ask_field("""Enter your AWS Region: [us-east-1]"""		,					default="""us-east-1"""     )
			SCREAMING_SNAKE_CASE__										=   aws_region

			SCREAMING_SNAKE_CASE__										=   _ask_options(
			    """Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"""		,					["""Provide IAM Role name""", """Create new IAM role using credentials"""]		,					snake_case__		,					)
			if role_management == 0:
						SCREAMING_SNAKE_CASE__										=   _ask_field("""Enter your IAM role name: """     )
			else:
						SCREAMING_SNAKE_CASE__										=   """accelerate_sagemaker_execution_role"""
						print(f"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""     )
						_create_iam_role_for_sagemaker(snake_case__     )

			SCREAMING_SNAKE_CASE__										=   _ask_field(
			    """Do you want to use custom Docker image? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
			SCREAMING_SNAKE_CASE__										=   None
			if is_custom_docker_image:
						SCREAMING_SNAKE_CASE__										=   _ask_field("""Enter your Docker image: """		,					lambda snake_case__     : str(snake_case__     ).lower()     )

			SCREAMING_SNAKE_CASE__										=   _ask_field(
			    """Do you want to provide SageMaker input channels with data locations? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
			SCREAMING_SNAKE_CASE__										=   None
			if is_sagemaker_inputs_enabled:
						SCREAMING_SNAKE_CASE__										=   _ask_field(
						    """Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): """		,					lambda snake_case__     : str(snake_case__     ).lower()		,					)

			SCREAMING_SNAKE_CASE__										=   _ask_field(
			    """Do you want to enable SageMaker metrics? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
			SCREAMING_SNAKE_CASE__										=   None
			if is_sagemaker_metrics_enabled:
						SCREAMING_SNAKE_CASE__										=   _ask_field(
						    """Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): """		,					lambda snake_case__     : str(snake_case__     ).lower()		,					)

			SCREAMING_SNAKE_CASE__										=   _ask_options(
			    """What is the distributed mode?"""		,					["""No distributed training""", """Data parallelism"""]		,					_convert_sagemaker_distributed_mode		,					)
			SCREAMING_SNAKE_CASE__										=   {}
			SCREAMING_SNAKE_CASE__										=   _ask_field(
			    """Do you wish to optimize your script with torch dynamo?[yes/NO]:"""		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
			if use_dynamo:
						SCREAMING_SNAKE_CASE__										=   """dynamo_"""
						SCREAMING_SNAKE_CASE__										=   _ask_options(
						    """Which dynamo backend would you like to use?"""		,					[x.lower() for x in DYNAMO_BACKENDS]		,					_convert_dynamo_backend		,					default=2		,					)
						SCREAMING_SNAKE_CASE__										=   _ask_field(
						    """Do you want to customize the defaults sent to torch.compile? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)

						if use_custom_options:
									SCREAMING_SNAKE_CASE__										=   _ask_options(
									    """Which mode do you want to use?"""		,					snake_case__		,					lambda snake_case__     : TORCH_DYNAMO_MODES[int(snake_case__     )]		,					default="""default"""		,					)
									SCREAMING_SNAKE_CASE__										=   _ask_field(
									    """Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
									SCREAMING_SNAKE_CASE__										=   _ask_field(
									    """Do you want to enable dynamic shape tracing? [yes/NO]: """		,					_convert_yes_no_to_bool		,					default=snake_case__		,					error_message="""Please enter yes or no."""		,					)
			SCREAMING_SNAKE_CASE__										=   """Which EC2 instance type you want to use for your training?"""
			if distributed_type != SageMakerDistributedType.NO:
						SCREAMING_SNAKE_CASE__										=   _ask_options(
						    snake_case__		,					snake_case__		,					lambda snake_case__     : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(snake_case__     )]     )
			else:
						eca_instance_query += "? [ml.p3.2xlarge]:"
						SCREAMING_SNAKE_CASE__										=   _ask_field(snake_case__		,					lambda snake_case__     : str(snake_case__     ).lower()		,					default="""ml.p3.2xlarge"""     )

			SCREAMING_SNAKE_CASE__										=   1
			if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
						SCREAMING_SNAKE_CASE__										=   _ask_field(
						    """How many machines do you want use? [1]: """		,					snake_case__		,					default=1		,					)

			SCREAMING_SNAKE_CASE__										=   _ask_options(
			    """Do you wish to use FP16 or BF16 (mixed precision)?"""		,					["""no""", """fp16""", """bf16""", """fp8"""]		,					_convert_mixed_precision		,					)

			if use_dynamo and mixed_precision == "no":
						print(
						    """Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."""     )

			return SageMakerConfig(
			    image_uri=snake_case__		,					compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER		,					distributed_type=snake_case__		,					use_cpu=snake_case__		,					dynamo_config=snake_case__		,					eca_instance_type=snake_case__		,					profile=snake_case__		,					region=snake_case__		,					iam_role_name=snake_case__		,					mixed_precision=snake_case__		,					num_machines=snake_case__		,					sagemaker_inputs_file=snake_case__		,					sagemaker_metrics_file=snake_case__		,					)





"""simple docstring"""




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		List[str]       		=				PhobertTokenizer
			lowerCamelCase__ :		Union[str, Any]       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Any:
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						SCREAMING_SNAKE_CASE__										=   ["""T@@""", """i""", """I""", """R@@""", """r""", """e@@"""]
						SCREAMING_SNAKE_CASE__										=   dict(zip(__UpperCAmelCase     ,					range(len(__UpperCAmelCase							)							)							)							)
						SCREAMING_SNAKE_CASE__										=   ["""#version: 0.2""", """l à</w>"""]
						SCREAMING_SNAKE_CASE__										=   {"""unk_token""": """<unk>"""}

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""merges_file"""]							)

						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									for token in vocab_tokens:
												fp.write(F"""{token} {vocab_tokens[token]}\n"""							)
						with open(self.merges_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									fp.write("""\n""".join(__UpperCAmelCase							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]     ,					**__UpperCAmelCase					:    int							) ->     Optional[int]:
						kwargs.update(self.special_tokens_map							)
						return PhobertTokenizer.from_pretrained(self.tmpdirname     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict     ,					__UpperCAmelCase					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
						return input_text, output_text





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   PhobertTokenizer(self.vocab_file     ,					self.merges_file     ,					**self.special_tokens_map							)
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						print(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokens + [tokenizer.unk_token]

						SCREAMING_SNAKE_CASE__										=   [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					__UpperCAmelCase							)


