


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
   import botoa  # noqa: F401






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  Any:




      """simple docstring"""

      snake_case_	=				botoa.client('''iam''' )

      snake_case_	=				{
          '''Version''': '''2012-10-17''',
          '''Statement''': [
              {'''Effect''': '''Allow''', '''Principal''': {'''Service''': '''sagemaker.amazonaws.com'''}, '''Action''': '''sts:AssumeRole'''}
          ],
      }
      try:
            # create the role, associated with the chosen trust policy
            iam_client.create_role(
                RoleName=SCREAMING_SNAKE_CASE , AssumeRolePolicyDocument=json.dumps(SCREAMING_SNAKE_CASE , indent=2 ) )
            snake_case_	=				{
                '''Version''': '''2012-10-17''',
                '''Statement''': [
                    {
                        '''Effect''': '''Allow''',
                        '''Action''': [
                            '''sagemaker:*''',
                            '''ecr:GetDownloadUrlForLayer''',
                            '''ecr:BatchGetImage''',
                            '''ecr:BatchCheckLayerAvailability''',
                            '''ecr:GetAuthorizationToken''',
                            '''cloudwatch:PutMetricData''',
                            '''cloudwatch:GetMetricData''',
                            '''cloudwatch:GetMetricStatistics''',
                            '''cloudwatch:ListMetrics''',
                            '''logs:CreateLogGroup''',
                            '''logs:CreateLogStream''',
                            '''logs:DescribeLogStreams''',
                            '''logs:PutLogEvents''',
                            '''logs:GetLogEvents''',
                            '''s3:CreateBucket''',
                            '''s3:ListBucket''',
                            '''s3:GetBucketLocation''',
                            '''s3:GetObject''',
                            '''s3:PutObject''',
                        ],
                        '''Resource''': '''*''',
                    }
                ],
            }
            # attach policy to role
            iam_client.put_role_policy(
                RoleName=SCREAMING_SNAKE_CASE , PolicyName=f'''{role_name}_policy_permission''' , PolicyDocument=json.dumps(SCREAMING_SNAKE_CASE , indent=2 ) , )
      except iam_client.exceptions.EntityAlreadyExistsException:
            print(f'''role {role_name} already exists. Using existing one''' )






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  Union[str, Any]:




      """simple docstring"""

      snake_case_	=				botoa.client('''iam''' )
      return iam_client.get_role(RoleName=SCREAMING_SNAKE_CASE )["Role"]["Arn"]






def 	__lowerCAmelCase   ()->  Any:




      """simple docstring"""

      snake_case_	=				_ask_options(
          '''How do you want to authorize?''' , ['''AWS Profile''', '''Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) '''] , SCREAMING_SNAKE_CASE , )
      snake_case_	=				None
      if credentials_configuration == 0:
            snake_case_	=				_ask_field('''Enter your AWS Profile name: [default] ''' , default='''default''' )
            snake_case_	=				aws_profile
      else:
            print(
                '''Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'''
                '''`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`''' )
            snake_case_	=				_ask_field('''AWS Access Key ID: ''' )
            snake_case_	=				aws_access_key_id

            snake_case_	=				_ask_field('''AWS Secret Access Key: ''' )
            snake_case_	=				aws_secret_access_key

      snake_case_	=				_ask_field('''Enter your AWS Region: [us-east-1]''' , default='''us-east-1''' )
      snake_case_	=				aws_region

      snake_case_	=				_ask_options(
          '''Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?''' , ['''Provide IAM Role name''', '''Create new IAM role using credentials'''] , SCREAMING_SNAKE_CASE , )
      if role_management == 0:
            snake_case_	=				_ask_field('''Enter your IAM role name: ''' )
      else:
            snake_case_	=				'''accelerate_sagemaker_execution_role'''
            print(f'''Accelerate will create an iam role "{iam_role_name}" using the provided credentials''' )
            _create_iam_role_for_sagemaker(SCREAMING_SNAKE_CASE )

      snake_case_	=				_ask_field(
          '''Do you want to use custom Docker image? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
      snake_case_	=				None
      if is_custom_docker_image:
            snake_case_	=				_ask_field('''Enter your Docker image: ''' , lambda SCREAMING_SNAKE_CASE : str(SCREAMING_SNAKE_CASE ).lower() )

      snake_case_	=				_ask_field(
          '''Do you want to provide SageMaker input channels with data locations? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
      snake_case_	=				None
      if is_sagemaker_inputs_enabled:
            snake_case_	=				_ask_field(
                '''Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): ''' , lambda SCREAMING_SNAKE_CASE : str(SCREAMING_SNAKE_CASE ).lower() , )

      snake_case_	=				_ask_field(
          '''Do you want to enable SageMaker metrics? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
      snake_case_	=				None
      if is_sagemaker_metrics_enabled:
            snake_case_	=				_ask_field(
                '''Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): ''' , lambda SCREAMING_SNAKE_CASE : str(SCREAMING_SNAKE_CASE ).lower() , )

      snake_case_	=				_ask_options(
          '''What is the distributed mode?''' , ['''No distributed training''', '''Data parallelism'''] , _convert_sagemaker_distributed_mode , )
      snake_case_	=				{}
      snake_case_	=				_ask_field(
          '''Do you wish to optimize your script with torch dynamo?[yes/NO]:''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
      if use_dynamo:
            snake_case_	=				'''dynamo_'''
            snake_case_	=				_ask_options(
                '''Which dynamo backend would you like to use?''' , [x.lower() for x in DYNAMO_BACKENDS] , _convert_dynamo_backend , default=2 , )
            snake_case_	=				_ask_field(
                '''Do you want to customize the defaults sent to torch.compile? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )

            if use_custom_options:
                  snake_case_	=				_ask_options(
                      '''Which mode do you want to use?''' , SCREAMING_SNAKE_CASE , lambda SCREAMING_SNAKE_CASE : TORCH_DYNAMO_MODES[int(SCREAMING_SNAKE_CASE )] , default='''default''' , )
                  snake_case_	=				_ask_field(
                      '''Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
                  snake_case_	=				_ask_field(
                      '''Do you want to enable dynamic shape tracing? [yes/NO]: ''' , _convert_yes_no_to_bool , default=SCREAMING_SNAKE_CASE , error_message='''Please enter yes or no.''' , )
      snake_case_	=				'''Which EC2 instance type you want to use for your training?'''
      if distributed_type != SageMakerDistributedType.NO:
            snake_case_	=				_ask_options(
                SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , lambda SCREAMING_SNAKE_CASE : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(SCREAMING_SNAKE_CASE )] )
      else:
            eca_instance_query += "? [ml.p3.2xlarge]:"
            snake_case_	=				_ask_field(SCREAMING_SNAKE_CASE , lambda SCREAMING_SNAKE_CASE : str(SCREAMING_SNAKE_CASE ).lower() , default='''ml.p3.2xlarge''' )

      snake_case_	=				1
      if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
            snake_case_	=				_ask_field(
                '''How many machines do you want use? [1]: ''' , SCREAMING_SNAKE_CASE , default=1 , )

      snake_case_	=				_ask_options(
          '''Do you wish to use FP16 or BF16 (mixed precision)?''' , ['''no''', '''fp16''', '''bf16''', '''fp8'''] , _convert_mixed_precision , )

      if use_dynamo and mixed_precision == "no":
            print(
                '''Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.''' )

      return SageMakerConfig(
          image_uri=SCREAMING_SNAKE_CASE , compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER , distributed_type=SCREAMING_SNAKE_CASE , use_cpu=SCREAMING_SNAKE_CASE , dynamo_config=SCREAMING_SNAKE_CASE , eca_instance_type=SCREAMING_SNAKE_CASE , profile=SCREAMING_SNAKE_CASE , region=SCREAMING_SNAKE_CASE , iam_role_name=SCREAMING_SNAKE_CASE , mixed_precision=SCREAMING_SNAKE_CASE , num_machines=SCREAMING_SNAKE_CASE , sagemaker_inputs_file=SCREAMING_SNAKE_CASE , sagemaker_metrics_file=SCREAMING_SNAKE_CASE , )



import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  Tuple:




      """simple docstring"""

      if (
          (cp >= 0X4E00 and cp <= 0X9FFF)
          or (cp >= 0X3400 and cp <= 0X4DBF)  #
          or (cp >= 0X2_0000 and cp <= 0X2_A6DF)  #
          or (cp >= 0X2_A700 and cp <= 0X2_B73F)  #
          or (cp >= 0X2_B740 and cp <= 0X2_B81F)  #
          or (cp >= 0X2_B820 and cp <= 0X2_CEAF)  #
          or (cp >= 0XF900 and cp <= 0XFAFF)
          or (cp >= 0X2_F800 and cp <= 0X2_FA1F)  #
      ):  #
            return True

      return False






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  List[Any]:




      """simple docstring"""

      for char in word:
            snake_case_	=				ord(SCREAMING_SNAKE_CASE )
            if not _is_chinese_char(SCREAMING_SNAKE_CASE ):
                  return 0
      return 1






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  str:




      """simple docstring"""

      snake_case_	=				set()

      for token in tokens:
            snake_case_	=				len(SCREAMING_SNAKE_CASE ) > 1 and is_chinese(SCREAMING_SNAKE_CASE )
            if chinese_word:
                  word_set.add(SCREAMING_SNAKE_CASE )
      snake_case_	=				list(SCREAMING_SNAKE_CASE )
      return word_list






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE )->  List[str]:




      """simple docstring"""

      if not chinese_word_set:
            return bert_tokens
      snake_case_	=				max([len(SCREAMING_SNAKE_CASE ) for w in chinese_word_set] )

      snake_case_	=				bert_tokens
      snake_case_	,   snake_case_	=				0, len(SCREAMING_SNAKE_CASE )
      while start < end:
            snake_case_	=				True
            if is_chinese(bert_word[start] ):
                  snake_case_	=				min(end - start , SCREAMING_SNAKE_CASE )
                  for i in range(SCREAMING_SNAKE_CASE , 1 , -1 ):
                        snake_case_	=				''''''.join(bert_word[start : start + i] )
                        if whole_word in chinese_word_set:
                              for j in range(start + 1 , start + i ):
                                    snake_case_	=				'''##''' + bert_word[j]
                              snake_case_	=				start + i
                              snake_case_	=				False
                              break
            if single_word:
                  start += 1
      return bert_word






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE )->  Any:




      """simple docstring"""

      snake_case_	=				[]

      for i in range(0 , len(SCREAMING_SNAKE_CASE ) , 100 ):
            snake_case_	=				ltp_tokenizer.pipeline(lines[i : i + 100] , tasks=['''cws'''] ).cws
            snake_case_	=				[get_chinese_word(SCREAMING_SNAKE_CASE ) for r in res]
            ltp_res.extend(SCREAMING_SNAKE_CASE )
      assert len(SCREAMING_SNAKE_CASE ) == len(SCREAMING_SNAKE_CASE )

      snake_case_	=				[]
      for i in range(0 , len(SCREAMING_SNAKE_CASE ) , 100 ):
            snake_case_	=				bert_tokenizer(lines[i : i + 100] , add_special_tokens=SCREAMING_SNAKE_CASE , truncation=SCREAMING_SNAKE_CASE , max_length=512 )
            bert_res.extend(res['''input_ids'''] )
      assert len(SCREAMING_SNAKE_CASE ) == len(SCREAMING_SNAKE_CASE )

      snake_case_	=				[]
      for input_ids, chinese_word in zip(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE ):
            snake_case_	=				[]
            for id in input_ids:
                  snake_case_	=				bert_tokenizer._convert_id_to_token(SCREAMING_SNAKE_CASE )
                  input_tokens.append(SCREAMING_SNAKE_CASE )
            snake_case_	=				add_sub_symbol(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE )
            snake_case_	=				[]
            # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
            for i, token in enumerate(SCREAMING_SNAKE_CASE ):
                  if token[:2] == "##":
                        snake_case_	=				token[2:]
                        # save chinese tokens' pos
                        if len(SCREAMING_SNAKE_CASE ) == 1 and _is_chinese_char(ord(SCREAMING_SNAKE_CASE ) ):
                              ref_id.append(SCREAMING_SNAKE_CASE )
            ref_ids.append(SCREAMING_SNAKE_CASE )

      assert len(SCREAMING_SNAKE_CASE ) == len(SCREAMING_SNAKE_CASE )

      return ref_ids






def 	__lowerCAmelCase   (SCREAMING_SNAKE_CASE )->  Tuple:




      """simple docstring"""

      with open(args.file_name , '''r''' , encoding='''utf-8''' ) as f:
            snake_case_	=				f.readlines()
      snake_case_	=				[line.strip() for line in data if len(SCREAMING_SNAKE_CASE ) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
      snake_case_	=				LTP(args.ltp )  # faster in GPU device
      snake_case_	=				BertTokenizer.from_pretrained(args.bert )

      snake_case_	=				prepare_ref(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE )

      with open(args.save_path , '''w''' , encoding='''utf-8''' ) as f:
            snake_case_	=				[json.dumps(SCREAMING_SNAKE_CASE ) + '''\n''' for ref in ref_ids]
            f.writelines(SCREAMING_SNAKE_CASE )


if __name__ == "__main__":
   UpperCAmelCase      =   argparse.ArgumentParser(description="""prepare_chinese_ref""")
   parser.add_argument(
       """--file_name""",
       required=False,
       type=str,
       default="""./resources/chinese-demo.txt""",
       help="""file need process, same as training data in lm""",
   )
   parser.add_argument(
       """--ltp""",
       required=False,
       type=str,
       default="""./resources/ltp""",
       help="""resources for LTP tokenizer, usually a path""",
   )
   parser.add_argument(
       """--bert""",
       required=False,
       type=str,
       default="""./resources/robert""",
       help="""resources for Bert tokenizer""",
   )
   parser.add_argument(
       """--save_path""",
       required=False,
       type=str,
       default="""./resources/ref.txt""",
       help="""path to save res""",
   )

   UpperCAmelCase      =   parser.parse_args()
   main(args)