
import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
		from sagemaker import Session, TrainingJobAnalytics
		from sagemaker.huggingface import HuggingFace







@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"		,	"False"	)	) is not True		,	reason="Skipping test because should only be run when releasing minor transformers version"		,	)
@pytest.mark.usefixtures("sm_env"	)
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue_model_parallelism.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1600, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "roberta-large",
            "instance_type": "ml.p3dn.24xlarge",
            "results": {"train_runtime": 1600, "eval_accuracy": 0.3, "eval_loss": 1.2},
        },
    ]	)
class        lowerCAmelCase__      (   unittest.TestCase	):

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								if self.framework == "pytorch":
										subprocess.run(
										    F'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split()       ,							encoding="""utf-8"""       ,							check=a       ,							)
								assert hasattr(self       ,							"""env"""				)

						def  A_				(			self       ,							a				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	{
								    """enabled""": True,
								    """processes_per_host""": 8,
								}
								_UpperCamelCase						    =	{
								    """enabled""": True,
								    """parameters""": {
								        """microbatches""": 4,
								        """placement_strategy""": """spread""",
								        """pipeline""": """interleaved""",
								        """optimize""": """speed""",
								        """partitions""": 4,
								        """ddp""": True,
								    },
								}

								_UpperCamelCase						    =	{"""smdistributed""": {"""modelparallel""": smp_options}, """mpi""": mpi_options}

								_UpperCamelCase						    =	"""trainer""" if self.script == """run_glue.py""" else """smtrainer"""
								# creates estimator
								return HuggingFace(
								    entry_point=self.script       ,							source_dir=self.env.test_path       ,							role=self.env.role       ,							image_uri=self.env.image_uri       ,							base_job_name=F'{self.env.base_job_name}-{instance_count}-smp-{name_extension}'       ,							instance_count=a       ,							instance_type=self.instance_type       ,							debugger_hook_config=a       ,							hyperparameters={
								        **self.env.hyperparameters,
								        """model_name_or_path""": self.model_name_or_path,
								        """max_steps""": 5_00,
								    }       ,							metric_definitions=self.env.metric_definitions       ,							distribution=a       ,							py_version="""py36"""       ,							)

						def  A_				(			self       ,							a				)					->      Union[str, Any]:


								'''simple docstring'''

								TrainingJobAnalytics(a				).export_csv(F'{self.env.test_path}/{job_name}_metrics.csv'				)







						@parameterized.expand([(1,)]				)
						def  A_				(			self       ,							a				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	self.create_estimator(a				)

								# run training
								estimator.fit()

								# result dataframe
								_UpperCamelCase						    =	TrainingJobAnalytics(estimator.latest_training_job.name				).dataframe()

								# extract kpis
								_UpperCamelCase						    =	list(result_metrics_df[result_metrics_df.metric_name == """eval_accuracy"""]["""value"""]				)
								_UpperCamelCase						    =	list(result_metrics_df[result_metrics_df.metric_name == """eval_loss"""]["""value"""]				)
								# get train time from SageMaker job, this includes starting, preprocessing, stopping
								_UpperCamelCase						    =	(
								    Session().describe_training_job(estimator.latest_training_job.name				).get("""TrainingTimeInSeconds"""       ,							99_99_99				)
								)

								# assert kpis
								assert train_runtime <= self.results["train_runtime"]
								assert all(t >= self.results["""eval_accuracy"""] for t in eval_accuracy				)
								assert all(t <= self.results["""eval_loss"""] for t in eval_loss				)

								# dump tests result into json file to share in PR
								with open(F'{estimator.latest_training_job.name}.json'       ,							"""w"""				) as outfile:
										json.dump({"""train_time""": train_runtime, """eval_accuracy""": eval_accuracy, """eval_loss""": eval_loss}       ,							a				)





import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase__	=							logging.get_logger(__name__)


def 							__A(lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase )		->				Any:

		"""simple docstring"""






		_UpperCamelCase						    =	os.path.abspath(lowerCAmelCase )
		logger.info(F'Converting TensorFlow checkpoint from {tf_path}' )
		# Load weights from TF model
		_UpperCamelCase						    =	tf.train.list_variables(lowerCAmelCase )
		_UpperCamelCase						    =	[]
		_UpperCamelCase						    =	[]
		_UpperCamelCase						    =	[]
		for full_name, shape in init_vars:
				# logger.info(f"Loading TF weight {name} with shape {shape}")
				_UpperCamelCase						    =	full_name.split("""/""" )
				if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
						logger.info(F'Skipping non-model layer {full_name}' )
						continue
				if "optimizer" in full_name:
						logger.info(F'Skipping optimization layer {full_name}' )
						continue
				if name[0] == "model":
						# ignore initial 'model'
						_UpperCamelCase						    =	name[1:]
				# figure out how many levels deep the name is
				_UpperCamelCase						    =	0
				for _name in name:
						if _name.startswith("""layer_with_weights""" ):
								depth += 1
						else:
								break
				layer_depth.append(lowerCAmelCase )
				# read data
				_UpperCamelCase						    =	tf.train.load_variable(lowerCAmelCase						,  lowerCAmelCase )
				names.append("""/""".join(lowerCAmelCase ) )
				arrays.append(lowerCAmelCase )
		logger.info(F'Read a total of {len(lowerCAmelCase ):,} layers' )

		# Sanity check
		if len(set(lowerCAmelCase ) ) != 1:
				raise ValueError(F'Found layer names with different depths (layer depth {list(set(lowerCAmelCase ) )})' )
		_UpperCamelCase						    =	list(set(lowerCAmelCase ) )[0]
		if layer_depth != 1:
				raise ValueError(
				    """The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"""
				    """ heads.""" )

		# convert layers
		logger.info("""Converting weights...""" )
		for full_name, array in zip(lowerCAmelCase						,  lowerCAmelCase ):
				_UpperCamelCase						    =	full_name.split("""/""" )
				_UpperCamelCase						    =	model
				_UpperCamelCase						    =	[]
				for i, m_name in enumerate(lowerCAmelCase ):
						if m_name == ".ATTRIBUTES":
								# variable names end with .ATTRIBUTES/VARIABLE_VALUE
								break
						if m_name.startswith("""layer_with_weights""" ):
								_UpperCamelCase						    =	int(m_name.split("""-""" )[-1] )
								if layer_num <= 2:
										# embedding layers
										# layer_num 0: word_embeddings
										# layer_num 1: position_embeddings
										# layer_num 2: token_type_embeddings
										continue
								elif layer_num == 3:
										# embedding LayerNorm
										trace.extend(["""embeddings""", """LayerNorm"""] )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """embeddings""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """LayerNorm""" )
								elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
										# encoder layers
										trace.extend(["""encoder""", """layer""", str(layer_num - 4 )] )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """encoder""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """layer""" )
										_UpperCamelCase						    =	pointer[layer_num - 4]
								elif layer_num == config.num_hidden_layers + 4:
										# pooler layer
										trace.extend(["""pooler""", """dense"""] )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """pooler""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """dense""" )
						elif m_name == "embeddings":
								trace.append("""embeddings""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """embeddings""" )
								if layer_num == 0:
										trace.append("""word_embeddings""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """word_embeddings""" )
								elif layer_num == 1:
										trace.append("""position_embeddings""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """position_embeddings""" )
								elif layer_num == 2:
										trace.append("""token_type_embeddings""" )
										_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """token_type_embeddings""" )
								else:
										raise ValueError(F'Unknown embedding layer with name {full_name}' )
								trace.append("""weight""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """weight""" )
						elif m_name == "_attention_layer":
								# self-attention layer
								trace.extend(["""attention""", """self"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """attention""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """self""" )
						elif m_name == "_attention_layer_norm":
								# output attention norm
								trace.extend(["""attention""", """output""", """LayerNorm"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """attention""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """output""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """LayerNorm""" )
						elif m_name == "_attention_output_dense":
								# output attention dense
								trace.extend(["""attention""", """output""", """dense"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """attention""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """output""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """dense""" )
						elif m_name == "_output_dense":
								# output dense
								trace.extend(["""output""", """dense"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """output""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """dense""" )
						elif m_name == "_output_layer_norm":
								# output dense
								trace.extend(["""output""", """LayerNorm"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """output""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """LayerNorm""" )
						elif m_name == "_key_dense":
								# attention key
								trace.append("""key""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """key""" )
						elif m_name == "_query_dense":
								# attention query
								trace.append("""query""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """query""" )
						elif m_name == "_value_dense":
								# attention value
								trace.append("""value""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """value""" )
						elif m_name == "_intermediate_dense":
								# attention intermediate dense
								trace.extend(["""intermediate""", """dense"""] )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """intermediate""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """dense""" )
						elif m_name == "_output_layer_norm":
								# output layer norm
								trace.append("""output""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """output""" )
						# weights & biases
						elif m_name in ["bias", "beta"]:
								trace.append("""bias""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """bias""" )
						elif m_name in ["kernel", "gamma"]:
								trace.append("""weight""" )
								_UpperCamelCase						    =	getattr(lowerCAmelCase						,  """weight""" )
						else:
								logger.warning(F'Ignored {m_name}' )
        # for certain layers reshape is necessary
				_UpperCamelCase						    =	""".""".join(lowerCAmelCase )
				if re.match(R"""(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)"""						,  lowerCAmelCase ) or re.match(
				    R"""(\S+)\.attention\.output\.dense\.weight"""						,  lowerCAmelCase ):
						_UpperCamelCase						    =	array.reshape(pointer.data.shape )
				if "kernel" in full_name:
						_UpperCamelCase						    =	array.transpose()
				if pointer.shape == array.shape:
						_UpperCamelCase						    =	torch.from_numpy(lowerCAmelCase )
				else:
						raise ValueError(
						    F'Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'
						    F' {array.shape}' )
				logger.info(F'Successfully set variable {full_name} to PyTorch layer {trace}' )
		return model


def 							__A(lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase )		->				Tuple:

		"""simple docstring"""






		logger.info(F'Loading model based on config from {config_path}...' )
		_UpperCamelCase						    =	BertConfig.from_json_file(lowerCAmelCase )
		_UpperCamelCase						    =	BertModel(lowerCAmelCase )

		# Load weights from checkpoint
		logger.info(F'Loading weights from checkpoint {tf_checkpoint_path}...' )
		load_tfa_weights_in_bert(lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase )

		# Save pytorch-model
		logger.info(F'Saving PyTorch model to {pytorch_dump_path}...' )
		torch.save(model.state_dict()						,  lowerCAmelCase )


if __name__ == "__main__":
		lowerCamelCase__	=							argparse.ArgumentParser()
		parser.add_argument(
		    "--tf_checkpoint_path", type=str, required=True, help="Path to the TensorFlow 2.x checkpoint path."
		)
		parser.add_argument(
		    "--bert_config_file",
		    type=str,
		    required=True,
		    help="The config json file corresponding to the BERT model. This specifies the model architecture.",
		)
		parser.add_argument(
		    "--pytorch_dump_path",
		    type=str,
		    required=True,
		    help="Path to the output PyTorch model (must include filename).",
		)
		lowerCamelCase__	=							parser.parse_args()
		convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)



