





# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
							import botoa  # noqa: F401





def      _lowercase					(						_UpperCAmelCase				)						->  Union[str, Any]:
				lowerCamelCase	 =botoa.client("""iam"""				)

				lowerCamelCase	 ={
				    """Version""": """2012-10-17""",
				    """Statement""": [
				        {"""Effect""": """Allow""", """Principal""": {"""Service""": """sagemaker.amazonaws.com"""}, """Action""": """sts:AssumeRole"""}
				    ],
				}
				try:
								# create the role, associated with the chosen trust policy
								iam_client.create_role(
								    RoleName=_UpperCAmelCase			,      AssumeRolePolicyDocument=json.dumps(_UpperCAmelCase			,      indent=2				)				)
								lowerCamelCase	 ={
								    """Version""": """2012-10-17""",
								    """Statement""": [
								        {
								            """Effect""": """Allow""",
								            """Action""": [
								                """sagemaker:*""",
								                """ecr:GetDownloadUrlForLayer""",
								                """ecr:BatchGetImage""",
								                """ecr:BatchCheckLayerAvailability""",
								                """ecr:GetAuthorizationToken""",
								                """cloudwatch:PutMetricData""",
								                """cloudwatch:GetMetricData""",
								                """cloudwatch:GetMetricStatistics""",
								                """cloudwatch:ListMetrics""",
								                """logs:CreateLogGroup""",
								                """logs:CreateLogStream""",
								                """logs:DescribeLogStreams""",
								                """logs:PutLogEvents""",
								                """logs:GetLogEvents""",
								                """s3:CreateBucket""",
								                """s3:ListBucket""",
								                """s3:GetBucketLocation""",
								                """s3:GetObject""",
								                """s3:PutObject""",
								            ],
								            """Resource""": """*""",
								        }
								    ],
								}
								# attach policy to role
								iam_client.put_role_policy(
								    RoleName=_UpperCAmelCase			,      PolicyName=F"""{role_name}_policy_permission"""			,      PolicyDocument=json.dumps(_UpperCAmelCase			,      indent=2				)			,      )
				except iam_client.exceptions.EntityAlreadyExistsException:
								print(F"""role {role_name} already exists. Using existing one"""				)





def      _lowercase					(						_UpperCAmelCase				)						->  Optional[Any]:
				lowerCamelCase	 =botoa.client("""iam"""				)
				return iam_client.get_role(RoleName=_UpperCAmelCase				)["Role"]["Arn"]





def      _lowercase					(						)						->  List[Any]:
				lowerCamelCase	 =_ask_options(
				    """How do you want to authorize?"""			,      ["""AWS Profile""", """Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) """]			,      _UpperCAmelCase			,      )
				lowerCamelCase	 =None
				if credentials_configuration == 0:
								lowerCamelCase	 =_ask_field("""Enter your AWS Profile name: [default] """			,      default="""default"""				)
								lowerCamelCase	 =aws_profile
				else:
								print(
								    """Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"""
								    """`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"""				)
								lowerCamelCase	 =_ask_field("""AWS Access Key ID: """				)
								lowerCamelCase	 =aws_access_key_id

								lowerCamelCase	 =_ask_field("""AWS Secret Access Key: """				)
								lowerCamelCase	 =aws_secret_access_key

				lowerCamelCase	 =_ask_field("""Enter your AWS Region: [us-east-1]"""			,      default="""us-east-1"""				)
				lowerCamelCase	 =aws_region

				lowerCamelCase	 =_ask_options(
				    """Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"""			,      ["""Provide IAM Role name""", """Create new IAM role using credentials"""]			,      _UpperCAmelCase			,      )
				if role_management == 0:
								lowerCamelCase	 =_ask_field("""Enter your IAM role name: """				)
				else:
								lowerCamelCase	 ="""accelerate_sagemaker_execution_role"""
								print(F"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""				)
								_create_iam_role_for_sagemaker(_UpperCAmelCase				)

				lowerCamelCase	 =_ask_field(
				    """Do you want to use custom Docker image? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
				lowerCamelCase	 =None
				if is_custom_docker_image:
								lowerCamelCase	 =_ask_field("""Enter your Docker image: """			,      lambda _UpperCAmelCase				: str(_UpperCAmelCase				).lower()				)

				lowerCamelCase	 =_ask_field(
				    """Do you want to provide SageMaker input channels with data locations? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
				lowerCamelCase	 =None
				if is_sagemaker_inputs_enabled:
								lowerCamelCase	 =_ask_field(
								    """Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): """			,      lambda _UpperCAmelCase				: str(_UpperCAmelCase				).lower()			,      )

				lowerCamelCase	 =_ask_field(
				    """Do you want to enable SageMaker metrics? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
				lowerCamelCase	 =None
				if is_sagemaker_metrics_enabled:
								lowerCamelCase	 =_ask_field(
								    """Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): """			,      lambda _UpperCAmelCase				: str(_UpperCAmelCase				).lower()			,      )

				lowerCamelCase	 =_ask_options(
				    """What is the distributed mode?"""			,      ["""No distributed training""", """Data parallelism"""]			,      _convert_sagemaker_distributed_mode			,      )
				lowerCamelCase	 ={}
				lowerCamelCase	 =_ask_field(
				    """Do you wish to optimize your script with torch dynamo?[yes/NO]:"""			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
				if use_dynamo:
								lowerCamelCase	 ="""dynamo_"""
								lowerCamelCase	 =_ask_options(
								    """Which dynamo backend would you like to use?"""			,      [x.lower() for x in DYNAMO_BACKENDS]			,      _convert_dynamo_backend			,      default=2			,      )
								lowerCamelCase	 =_ask_field(
								    """Do you want to customize the defaults sent to torch.compile? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )

								if use_custom_options:
												lowerCamelCase	 =_ask_options(
												    """Which mode do you want to use?"""			,      _UpperCAmelCase			,      lambda _UpperCAmelCase				: TORCH_DYNAMO_MODES[int(_UpperCAmelCase				)]			,      default="""default"""			,      )
												lowerCamelCase	 =_ask_field(
												    """Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
												lowerCamelCase	 =_ask_field(
												    """Do you want to enable dynamic shape tracing? [yes/NO]: """			,      _convert_yes_no_to_bool			,      default=_UpperCAmelCase			,      error_message="""Please enter yes or no."""			,      )
				lowerCamelCase	 ="""Which EC2 instance type you want to use for your training?"""
				if distributed_type != SageMakerDistributedType.NO:
								lowerCamelCase	 =_ask_options(
								    _UpperCAmelCase			,      _UpperCAmelCase			,      lambda _UpperCAmelCase				: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(_UpperCAmelCase				)]				)
				else:
								eca_instance_query += "? [ml.p3.2xlarge]:"
								lowerCamelCase	 =_ask_field(_UpperCAmelCase			,      lambda _UpperCAmelCase				: str(_UpperCAmelCase				).lower()			,      default="""ml.p3.2xlarge"""				)

				lowerCamelCase	 =1
				if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
								lowerCamelCase	 =_ask_field(
								    """How many machines do you want use? [1]: """			,      _UpperCAmelCase			,      default=1			,      )

				lowerCamelCase	 =_ask_options(
				    """Do you wish to use FP16 or BF16 (mixed precision)?"""			,      ["""no""", """fp16""", """bf16""", """fp8"""]			,      _convert_mixed_precision			,      )

				if use_dynamo and mixed_precision == "no":
								print(
								    """Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."""				)

				return SageMakerConfig(
				    image_uri=_UpperCAmelCase			,      compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER			,      distributed_type=_UpperCAmelCase			,      use_cpu=_UpperCAmelCase			,      dynamo_config=_UpperCAmelCase			,      eca_instance_type=_UpperCAmelCase			,      profile=_UpperCAmelCase			,      region=_UpperCAmelCase			,      iam_role_name=_UpperCAmelCase			,      mixed_precision=_UpperCAmelCase			,      num_machines=_UpperCAmelCase			,      sagemaker_inputs_file=_UpperCAmelCase			,      sagemaker_metrics_file=_UpperCAmelCase			,      )












import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase__						:			Optional[Any]									=logging.get_logger(__name__)

UpperCAmelCase__						:			Union[str, Any]									={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__						:			str									={
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__						:			Union[str, Any]									={
    '''Salesforce/codegen-350M-mono''': 20_48,
}

class       __A     (						a    ):
				__A      =				VOCAB_FILES_NAMES
				__A      =				PRETRAINED_VOCAB_FILES_MAP
				__A      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A      =				["""input_ids""", """attention_mask"""]
				__A      =				CodeGenTokenizer


				def __init__(		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_=False   , **UpperCAmelCase_   , ):
								super().__init__(
								    UpperCAmelCase_   , UpperCAmelCase_   , tokenizer_file=UpperCAmelCase_   , unk_token=UpperCAmelCase_   , bos_token=UpperCAmelCase_   , eos_token=UpperCAmelCase_   , add_prefix_space=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if kwargs.pop("""add_bos_token"""   , UpperCAmelCase_							):
												lowerCamelCase	 =kwargs.pop("""name_or_path"""   , """"""							)
												raise ValueError(
												    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
												    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
												    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
												    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
												    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
												    """ so that the fast tokenizer works correctly."""							)

								lowerCamelCase	 =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
								if pre_tok_state.get("""add_prefix_space"""   , UpperCAmelCase_							) != add_prefix_space:
												lowerCamelCase	 =getattr(UpperCAmelCase_   , pre_tok_state.pop("""type"""							)							)
												lowerCamelCase	 =add_prefix_space
												lowerCamelCase	 =pre_tok_class(**UpperCAmelCase_							)

								lowerCamelCase	 =add_prefix_space


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = None							):
								lowerCamelCase	 =self._tokenizer.model.save(UpperCAmelCase_   , name=UpperCAmelCase_							)
								return tuple(UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = False   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , **UpperCAmelCase_   , ):
								lowerCamelCase	 =super().decode(
								    token_ids=UpperCAmelCase_   , skip_special_tokens=UpperCAmelCase_   , clean_up_tokenization_spaces=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if truncate_before_pattern is not None and len(UpperCAmelCase_							) > 0:
												lowerCamelCase	 =self.truncate(UpperCAmelCase_   , UpperCAmelCase_							)

								return decoded_text


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_							):
								def find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							):
												lowerCamelCase	 =pattern.search(UpperCAmelCase_   , UpperCAmelCase_							)
												return m.start() if m else -1

								lowerCamelCase	 =[re.compile(UpperCAmelCase_   , re.MULTILINE							) for pattern in truncate_before_pattern]

								lowerCamelCase	 =list(re.finditer("""^print"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: prints[1].start()]

								lowerCamelCase	 =list(re.finditer("""^def"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: defs[1].start()]

								lowerCamelCase	 =0

								lowerCamelCase	 =[
								    pos for pos in [find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							) for terminal in terminals] if pos != -1
								]

								if len(UpperCAmelCase_							) > 0:
												return completion[: min(UpperCAmelCase_							)]
								else:
												return completion





