



'''simple docstring'''





import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
  import botoa  # noqa: F401




def lowerCAmelCase						(__A):
     _a											=						botoa.client('''iam''')

     _a											=						{
         '''Version''': '''2012-10-17''',
         '''Statement''': [
             {'''Effect''': '''Allow''', '''Principal''': {'''Service''': '''sagemaker.amazonaws.com'''}, '''Action''': '''sts:AssumeRole'''}
         ],
     }
     try:
          # create the role, associated with the chosen trust policy
          iam_client.create_role(
              RoleName=__A						,			AssumeRolePolicyDocument=json.dumps(__A						,			indent=2))
          _a											=						{
              '''Version''': '''2012-10-17''',
              '''Statement''': [
                  {
                      '''Effect''': '''Allow''',
                      '''Action''': [
                          '''sagemaker:*''',
                          '''ecr:GetDownloadUrlForLayer''',
                          '''ecr:BatchGetImage''',
                          '''ecr:BatchCheckLayerAvailability''',
                          '''ecr:GetAuthorizationToken''',
                          '''cloudwatch:PutMetricData''',
                          '''cloudwatch:GetMetricData''',
                          '''cloudwatch:GetMetricStatistics''',
                          '''cloudwatch:ListMetrics''',
                          '''logs:CreateLogGroup''',
                          '''logs:CreateLogStream''',
                          '''logs:DescribeLogStreams''',
                          '''logs:PutLogEvents''',
                          '''logs:GetLogEvents''',
                          '''s3:CreateBucket''',
                          '''s3:ListBucket''',
                          '''s3:GetBucketLocation''',
                          '''s3:GetObject''',
                          '''s3:PutObject''',
                      ],
                      '''Resource''': '''*''',
                  }
              ],
          }
          # attach policy to role
          iam_client.put_role_policy(
              RoleName=__A						,			PolicyName=F'''{role_name}_policy_permission'''						,			PolicyDocument=json.dumps(__A						,			indent=2)						,			)
     except iam_client.exceptions.EntityAlreadyExistsException:
          print(F'''role {role_name} already exists. Using existing one''')




def lowerCAmelCase						(__A):
     _a											=						botoa.client('''iam''')
     return iam_client.get_role(RoleName=__A)["Role"]["Arn"]




def lowerCAmelCase						():
     _a											=						_ask_options(
         '''How do you want to authorize?'''						,			['''AWS Profile''', '''Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) ''']						,			__A						,			)
     _a											=						None
     if credentials_configuration == 0:
          _a											=						_ask_field('''Enter your AWS Profile name: [default] '''						,			default='''default''')
          _a											=						aws_profile
     else:
          print(
              '''Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'''
              '''`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`''')
          _a											=						_ask_field('''AWS Access Key ID: ''')
          _a											=						aws_access_key_id

          _a											=						_ask_field('''AWS Secret Access Key: ''')
          _a											=						aws_secret_access_key

     _a											=						_ask_field('''Enter your AWS Region: [us-east-1]'''						,			default='''us-east-1''')
     _a											=						aws_region

     _a											=						_ask_options(
         '''Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?'''						,			['''Provide IAM Role name''', '''Create new IAM role using credentials''']						,			__A						,			)
     if role_management == 0:
          _a											=						_ask_field('''Enter your IAM role name: ''')
     else:
          _a											=						'''accelerate_sagemaker_execution_role'''
          print(F'''Accelerate will create an iam role "{iam_role_name}" using the provided credentials''')
          _create_iam_role_for_sagemaker(__A)

     _a											=						_ask_field(
         '''Do you want to use custom Docker image? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
     _a											=						None
     if is_custom_docker_image:
          _a											=						_ask_field('''Enter your Docker image: '''						,			lambda __A: str(__A).lower())

     _a											=						_ask_field(
         '''Do you want to provide SageMaker input channels with data locations? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
     _a											=						None
     if is_sagemaker_inputs_enabled:
          _a											=						_ask_field(
              '''Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): '''						,			lambda __A: str(__A).lower()						,			)

     _a											=						_ask_field(
         '''Do you want to enable SageMaker metrics? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
     _a											=						None
     if is_sagemaker_metrics_enabled:
          _a											=						_ask_field(
              '''Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): '''						,			lambda __A: str(__A).lower()						,			)

     _a											=						_ask_options(
         '''What is the distributed mode?'''						,			['''No distributed training''', '''Data parallelism''']						,			_convert_sagemaker_distributed_mode						,			)
     _a											=						{}
     _a											=						_ask_field(
         '''Do you wish to optimize your script with torch dynamo?[yes/NO]:'''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
     if use_dynamo:
          _a											=						'''dynamo_'''
          _a											=						_ask_options(
              '''Which dynamo backend would you like to use?'''						,			[x.lower() for x in DYNAMO_BACKENDS]						,			_convert_dynamo_backend						,			default=2						,			)
          _a											=						_ask_field(
              '''Do you want to customize the defaults sent to torch.compile? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)

          if use_custom_options:
               _a											=						_ask_options(
                   '''Which mode do you want to use?'''						,			__A						,			lambda __A: TORCH_DYNAMO_MODES[int(__A)]						,			default='''default'''						,			)
               _a											=						_ask_field(
                   '''Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
               _a											=						_ask_field(
                   '''Do you want to enable dynamic shape tracing? [yes/NO]: '''						,			_convert_yes_no_to_bool						,			default=__A						,			error_message='''Please enter yes or no.'''						,			)
     _a											=						'''Which EC2 instance type you want to use for your training?'''
     if distributed_type != SageMakerDistributedType.NO:
          _a											=						_ask_options(
              __A						,			__A						,			lambda __A: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(__A)])
     else:
          eca_instance_query += "? [ml.p3.2xlarge]:"
          _a											=						_ask_field(__A						,			lambda __A: str(__A).lower()						,			default='''ml.p3.2xlarge''')

     _a											=						1
     if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
          _a											=						_ask_field(
              '''How many machines do you want use? [1]: '''						,			__A						,			default=1						,			)

     _a											=						_ask_options(
         '''Do you wish to use FP16 or BF16 (mixed precision)?'''						,			['''no''', '''fp16''', '''bf16''', '''fp8''']						,			_convert_mixed_precision						,			)

     if use_dynamo and mixed_precision == "no":
          print(
              '''Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.''')

     return SageMakerConfig(
         image_uri=__A						,			compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER						,			distributed_type=__A						,			use_cpu=__A						,			dynamo_config=__A						,			eca_instance_type=__A						,			profile=__A						,			region=__A						,			iam_role_name=__A						,			mixed_precision=__A						,			num_machines=__A						,			sagemaker_inputs_file=__A						,			sagemaker_metrics_file=__A						,			)








'''simple docstring'''





from __future__ import annotations

import unittest

from transformers import EsmConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import numpy
  import tensorflow as tf

  from transformers.models.esm.modeling_tf_esm import (
      TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFEsmForMaskedLM,
      TFEsmForSequenceClassification,
      TFEsmForTokenClassification,
      TFEsmModel,
  )






class __A    :



 '''simple docstring'''







 def __init__(self				,     A				,     )					->  Optional[int]:


      """simple docstring"""



      _a											=						parent
      _a											=						13
      _a											=						7
      _a											=						True
      _a											=						True
      _a											=						True
      _a											=						99
      _a											=						32
      _a											=						2
      _a											=						4
      _a											=						37
      _a											=						'''gelu'''
      _a											=						0.1
      _a											=						0.1
      _a											=						512
      _a											=						16
      _a											=						2
      _a											=						0.02
      _a											=						3
      _a											=						4
      _a											=						None



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size )

      _a											=						None
      if self.use_input_mask:
           _a											=						random_attention_mask([self.batch_size, self.seq_length] )

      _a											=						None
      _a											=						None
      _a											=						None
      if self.use_labels:
           _a											=						ids_tensor([self.batch_size]				,     self.type_sequence_label_size )
           _a											=						ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels )
           _a											=						ids_tensor([self.batch_size]				,     self.num_choices )

      _a											=						EsmConfig(
          vocab_size=self.vocab_size				,     hidden_size=self.hidden_size				,     num_hidden_layers=self.num_hidden_layers				,     pad_token_id=1				,     num_attention_heads=self.num_attention_heads				,     intermediate_size=self.intermediate_size				,     hidden_act=self.hidden_act				,     hidden_dropout_prob=self.hidden_dropout_prob				,     attention_probs_dropout_prob=self.attention_probs_dropout_prob				,     max_position_embeddings=self.max_position_embeddings				,     type_vocab_size=self.type_vocab_size				,     initializer_range=self.initializer_range				,     )

      return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



 def       a__						(self )					->  str:


      """simple docstring"""



      (
          (
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					
      )											=						self.prepare_config_and_inputs()

      _a											=						True
      _a											=						floats_tensor([self.batch_size, self.seq_length, self.hidden_size] )
      _a											=						ids_tensor([self.batch_size, self.seq_length]				,     vocab_size=2 )

      return (
          config,
          input_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      )



 def       a__						(self				,     A				,     A				,     A				,     A				,     A				,     A )					->  str:


      """simple docstring"""



      _a											=						TFEsmModel(config=A )
      _a											=						{'''input_ids''': input_ids, '''attention_mask''': input_mask}
      _a											=						model(A )

      _a											=						[input_ids, input_mask]
      _a											=						model(A )

      _a											=						model(A )

      self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size) )



 def       a__						(self				,     A				,     A				,     A				,     A				,     A				,     A				,     A				,     A				,     )					->  Union[str, Any]:


      """simple docstring"""



      _a											=						True

      _a											=						TFEsmModel(config=A )
      _a											=						{
          '''input_ids''': input_ids,
          '''attention_mask''': input_mask,
          '''encoder_hidden_states''': encoder_hidden_states,
          '''encoder_attention_mask''': encoder_attention_mask,
      }
      _a											=						model(A )

      _a											=						[input_ids, input_mask]
      _a											=						model(A				,     encoder_hidden_states=A )

      # Also check the case where encoder outputs are not passed
      _a											=						model(A				,     attention_mask=A )

      self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size) )



 def       a__						(self				,     A				,     A				,     A				,     A				,     A				,     A )					->  Optional[Any]:


      """simple docstring"""



      _a											=						TFEsmForMaskedLM(config=A )
      _a											=						model([input_ids, input_mask] )
      self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.vocab_size) )



 def       a__						(self				,     A				,     A				,     A				,     A				,     A				,     A )					->  Any:


      """simple docstring"""



      _a											=						self.num_labels
      _a											=						TFEsmForTokenClassification(config=A )
      _a											=						{'''input_ids''': input_ids, '''attention_mask''': input_mask}
      _a											=						model(A )
      self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.num_labels) )



 def       a__						(self )					->  int:


      """simple docstring"""



      _a											=						self.prepare_config_and_inputs()
      (
          (
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					(
          _a
      )						,					
      )											=						config_and_inputs
      _a											=						{'''input_ids''': input_ids, '''attention_mask''': input_mask}
      return config, inputs_dict






@require_tf
class __A    (						A     ,     A     ,     unittest.TestCase		):



 '''simple docstring'''







 __lowerCamelCase					: str    = (
     (
         TFEsmModel,
         TFEsmForMaskedLM,
         TFEsmForSequenceClassification,
         TFEsmForTokenClassification,
     )
     if is_tf_available()
     else ()
 )
 __lowerCamelCase					: Union[str, Any]    = (
     {
         'feature-extraction': TFEsmModel,
         'fill-mask': TFEsmForMaskedLM,
         'text-classification': TFEsmForSequenceClassification,
         'token-classification': TFEsmForTokenClassification,
         'zero-shot': TFEsmForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 __lowerCamelCase					: Dict    = False
 __lowerCamelCase					: Tuple    = False



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						TFEsmModelTester(self )
      _a											=						ConfigTester(self				,     config_class=A				,     hidden_size=37 )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      self.config_tester.run_common_tests()



 def       a__						(self )					->  Tuple:


      """simple docstring"""



      _a											=						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*A )



 def       a__						(self )					->  Dict:


      """simple docstring"""



      _a											=						self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_model_as_decoder(*A )



 def       a__						(self )					->  Tuple:


      """simple docstring"""



      _a											=						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*A )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      _a											=						self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*A )



 @slow
 def       a__						(self )					->  Tuple:


      """simple docstring"""



      for model_name in TF_ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           _a											=						TFEsmModel.from_pretrained(A )
           self.assertIsNotNone(A )



 @unittest.skip('''Protein models do not support embedding resizing.''' )
 def       a__						(self )					->  str:


      """simple docstring"""



      pass



 @unittest.skip('''Protein models do not support embedding resizing.''' )
 def       a__						(self )					->  Tuple:


      """simple docstring"""



      pass



 def       a__						(self )					->  Dict:


      """simple docstring"""



      _a						,					_a											=						self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
           _a											=						model_class(A )
           assert isinstance(model.get_input_embeddings()				,     tf.keras.layers.Layer )
           if model_class is TFEsmForMaskedLM:
                # Output embedding test differs from the main test because they're a matrix, not a layer
                _a											=						model.get_bias()
                assert isinstance(A				,     A )
                for k, v in name.items():
                     assert isinstance(A				,     tf.Variable )
           else:
                _a											=						model.get_output_embeddings()
                assert x is None
                _a											=						model.get_bias()
                assert name is None






@require_tf
class __A    (						unittest.TestCase		):



 '''simple docstring'''







 @slow
 def       a__						(self )					->  List[str]:


      """simple docstring"""



      _a											=						TFEsmForMaskedLM.from_pretrained('''facebook/esm2_t6_8M_UR50D''' )

      _a											=						tf.constant([[0, 1, 2, 3, 4, 5]] )
      _a											=						model(A )[0]
      _a											=						[1, 6, 33]
      self.assertEqual(list(output.numpy().shape )				,     A )
      # compare the actual values for a slice.
      _a											=						tf.constant(
          [
              [
                  [8.921518, -10.589814, -6.4671307],
                  [-6.3967156, -13.911377, -1.1211915],
                  [-7.781247, -13.951557, -3.740592],
              ]
          ] )
      self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()				,     expected_slice.numpy()				,     atol=1E-2 ) )



 @slow
 def       a__						(self )					->  Tuple:


      """simple docstring"""



      _a											=						TFEsmModel.from_pretrained('''facebook/esm2_t6_8M_UR50D''' )

      _a											=						tf.constant([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]] )
      _a											=						model(A )[0]
      # compare the actual values for a slice.
      _a											=						tf.constant(
          [
              [
                  [0.14443092, 0.54125327, 0.3247739],
                  [0.30340484, 0.00526676, 0.31077722],
                  [0.32278043, -0.24987096, 0.3414628],
              ]
          ] )
      self.assertTrue(numpy.allclose(output[:, :3, :3].numpy()				,     expected_slice.numpy()				,     atol=1E-4 ) )



