



"""simple docstring"""





import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
  from sagemaker import Session, TrainingJobAnalytics
  from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''							,   '''False''' ) ) is not True							,   reason='''Skipping test because should only be run when releasing minor transformers version'''							,   )
@pytest.mark.usefixtures('''sm_env''' )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue_model_parallelism.py''',
            '''model_name_or_path''': '''roberta-large''',
            '''instance_type''': '''ml.p3dn.24xlarge''',
            '''results''': {'''train_runtime''': 16_00, '''eval_accuracy''': 0.3, '''eval_loss''': 1.2},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''roberta-large''',
            '''instance_type''': '''ml.p3dn.24xlarge''',
            '''results''': {'''train_runtime''': 16_00, '''eval_accuracy''': 0.3, '''eval_loss''': 1.2},
        },
    ] )
class  lowercase__  (       unittest.TestCase ):







       '''simple docstring'''


       def 	lowercase__							(				self     :    int       )					->		List[Any]:


             '''simple docstring'''




             if self.framework == "pytorch":
                   subprocess.run(
                       F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()						,							encoding="utf-8"						,							check=__lowerCamelCase						,							)
             assert hasattr(self						,							"env"       )



       def 	lowercase__							(				self     :    List[Any]						,							_UpperCAmelCase     :    Dict       )					->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				{
                 "enabled": True,
                 "processes_per_host": 8,
             }
             UpperCAmelCase_     					=				{
                 "enabled": True,
                 "parameters": {
                     "microbatches": 4,
                     "placement_strategy": "spread",
                     "pipeline": "interleaved",
                     "optimize": "speed",
                     "partitions": 4,
                     "ddp": True,
                 },
             }

             UpperCAmelCase_     					=				{"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

             UpperCAmelCase_     					=				"trainer" if self.script == "run_glue.py" else "smtrainer"
             # creates estimator
             return HuggingFace(
                 entry_point=self.script						,							source_dir=self.env.test_path						,							role=self.env.role						,							image_uri=self.env.image_uri						,							base_job_name=F"""{self.env.base_job_name}-{instance_count}-smp-{name_extension}"""						,							instance_count=__lowerCamelCase						,							instance_type=self.instance_type						,							debugger_hook_config=__lowerCamelCase						,							hyperparameters={
                     **self.env.hyperparameters,
                     "model_name_or_path": self.model_name_or_path,
                     "max_steps": 500,
                 }						,							metric_definitions=self.env.metric_definitions						,							distribution=__lowerCamelCase						,							py_version="py36"						,							)



       def 	lowercase__							(				self     :    Tuple						,							_UpperCAmelCase     :    str       )					->		Optional[int]:


             '''simple docstring'''




             TrainingJobAnalytics(__lowerCamelCase       ).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""       )







       @parameterized.expand([(1,)]       )
       def 	lowercase__							(				self     :    Dict						,							_UpperCAmelCase     :    int       )					->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_     					=				self.create_estimator(__lowerCamelCase       )

             # run training
             estimator.fit()

             # result dataframe
             UpperCAmelCase_     					=				TrainingJobAnalytics(estimator.latest_training_job.name       ).dataframe()

             # extract kpis
             UpperCAmelCase_     					=				list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]       )
             UpperCAmelCase_     					=				list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]       )
             # get train time from SageMaker job, this includes starting, preprocessing, stopping
             UpperCAmelCase_     					=				(
                 Session().describe_training_job(estimator.latest_training_job.name       ).get("TrainingTimeInSeconds"						,							999999       )
             )

             # assert kpis
             assert train_runtime <= self.results["train_runtime"]
             assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy       )
             assert all(t <= self.results["eval_loss"] for t in eval_loss       )

             # dump tests result into json file to share in PR
             with open(F"""{estimator.latest_training_job.name}.json"""						,							"w"       ) as outfile:
                   json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}						,							__lowerCamelCase       )








import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[Any]:
   UpperCAmelCase      					=  ("""dense.weight""", """attention.self.query""", """attention.self.key""", """attention.self.value""")

   UpperCAmelCase      					=  (
       ("""layer.""", """layer_"""),
       ("""word_embeddings.weight""", """word_embeddings"""),
       ("""position_embeddings.weight""", """position_embeddings"""),
       ("""token_type_embeddings.weight""", """token_type_embeddings"""),
       (""".""", """/"""),
       ("""LayerNorm/weight""", """LayerNorm/gamma"""),
       ("""LayerNorm/bias""", """LayerNorm/beta"""),
       ("""weight""", """kernel"""),
   )

   if not os.path.isdir(lowerCAmelCase_ ):
      os.makedirs(lowerCAmelCase_ )

   UpperCAmelCase      					=  model.state_dict()

   def to_tf_var_name(lowerCAmelCase_ ):
      for patt, repl in iter(lowerCAmelCase_ ):
         UpperCAmelCase      					=  name.replace(lowerCAmelCase_	,    lowerCAmelCase_ )
      return F"""bert/{name}"""

   def create_tf_var(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ ):
      UpperCAmelCase      					=  tf.dtypes.as_dtype(tensor.dtype )
      UpperCAmelCase      					=  tf.get_variable(dtype=lowerCAmelCase_	,    shape=tensor.shape	,    name=lowerCAmelCase_	,    initializer=tf.zeros_initializer() )
      session.run(tf.variables_initializer([tf_var] ) )
      session.run(lowerCAmelCase_ )
      return tf_var

   tf.reset_default_graph()
   with tf.Session() as session:
      for var_name in state_dict:
         UpperCAmelCase      					=  to_tf_var_name(lowerCAmelCase_ )
         UpperCAmelCase      					=  state_dict[var_name].numpy()
         if any(x in var_name for x in tensors_to_transpose ):
            UpperCAmelCase      					=  torch_tensor.T
         UpperCAmelCase      					=  create_tf_var(tensor=lowerCAmelCase_	,    name=lowerCAmelCase_	,    session=lowerCAmelCase_ )
         tf.keras.backend.set_value(lowerCAmelCase_	,    lowerCAmelCase_ )
         UpperCAmelCase      					=  session.run(lowerCAmelCase_ )
         print(F"""Successfully created {tf_name}: {np.allclose(lowerCAmelCase_	,    lowerCAmelCase_ )}""" )

      UpperCAmelCase      					=  tf.train.Saver(tf.trainable_variables() )
      saver.save(lowerCAmelCase_	,    os.path.join(lowerCAmelCase_	,    model_name.replace("""-"""	,    """_""" ) + """.ckpt""" ) )
def 	_UpperCamelCase					(    lowerCAmelCase_=None )    ->Union[str, Any]:
   UpperCAmelCase      					=  argparse.ArgumentParser()
   parser.add_argument("""--model_name"""	,    type=lowerCAmelCase_	,    required=lowerCAmelCase_	,    help="""model name e.g. bert-base-uncased""" )
   parser.add_argument(
       """--cache_dir"""	,    type=lowerCAmelCase_	,    default=lowerCAmelCase_	,    required=lowerCAmelCase_	,    help="""Directory containing pytorch model""" )
   parser.add_argument("""--pytorch_model_path"""	,    type=lowerCAmelCase_	,    required=lowerCAmelCase_	,    help="""/path/to/<pytorch-model-name>.bin""" )
   parser.add_argument("""--tf_cache_dir"""	,    type=lowerCAmelCase_	,    required=lowerCAmelCase_	,    help="""Directory in which to save tensorflow model""" )
   UpperCAmelCase      					=  parser.parse_args(lowerCAmelCase_ )

   UpperCAmelCase      					=  BertModel.from_pretrained(
       pretrained_model_name_or_path=args.model_name	,    state_dict=torch.load(args.pytorch_model_path )	,    cache_dir=args.cache_dir	,    )

   convert_pytorch_checkpoint_to_tf(model=lowerCAmelCase_	,    ckpt_dir=args.tf_cache_dir	,    model_name=args.model_name )


if __name__ == "__main__":
 main()



