




import argparse

import tensorflow as tf
import torch

from transformers import BertConfig, BertForMaskedLM
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertPooler,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


logging.set_verbosity_info()


def 		lowerCamelCase__       (						_A	,      _A	,      _A				):







	'''simple docstring'''





	def get_masked_lm_array(_A				):
		snake_case_						       =	f"masked_lm/{name}/.ATTRIBUTES/VARIABLE_VALUE"
		snake_case_						       =	tf.train.load_variable(_A	,      _A				)

		if "kernel" in name:
			snake_case_						       =	array.transpose()

		return torch.from_numpy(_A				)

	def get_encoder_array(_A				):
		snake_case_						       =	f"encoder/{name}/.ATTRIBUTES/VARIABLE_VALUE"
		snake_case_						       =	tf.train.load_variable(_A	,      _A				)

		if "kernel" in name:
			snake_case_						       =	array.transpose()

		return torch.from_numpy(_A				)

	def get_encoder_layer_array(_A	,      _A				):
		snake_case_						       =	f"encoder/_transformer_layers/{layer_index}/{name}/.ATTRIBUTES/VARIABLE_VALUE"
		snake_case_						       =	tf.train.load_variable(_A	,      _A				)

		if "kernel" in name:
			snake_case_						       =	array.transpose()

		return torch.from_numpy(_A				)

	def get_encoder_attention_layer_array(_A	,      _A	,      _A				):
		snake_case_						       =	f"encoder/_transformer_layers/{layer_index}/_attention_layer/{name}/.ATTRIBUTES/VARIABLE_VALUE"
		snake_case_						       =	tf.train.load_variable(_A	,      _A				)
		snake_case_						       =	array.reshape(_A				)

		if "kernel" in name:
			snake_case_						       =	array.transpose()

		return torch.from_numpy(_A				)

	print(f"Loading model based on config from {config_path}..."				)
	snake_case_						       =	BertConfig.from_json_file(_A				)
	snake_case_						       =	BertForMaskedLM(_A				)

	# Layers
	for layer_index in range(0	,      config.num_hidden_layers				):
		snake_case_						       =	model.bert.encoder.layer[layer_index]

		# Self-attention
		snake_case_						       =	layer.attention.self

		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_query_dense/kernel"	,      self_attn.query.weight.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_query_dense/bias"	,      self_attn.query.bias.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_key_dense/kernel"	,      self_attn.key.weight.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_key_dense/bias"	,      self_attn.key.bias.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_value_dense/kernel"	,      self_attn.value.weight.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_value_dense/bias"	,      self_attn.value.bias.data.shape				)

		# Self-attention Output
		snake_case_						       =	layer.attention.output

		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_output_dense/kernel"	,      self_output.dense.weight.data.shape				)
		snake_case_						       =	get_encoder_attention_layer_array(
		    _A	,      "_output_dense/bias"	,      self_output.dense.bias.data.shape				)

		snake_case_						       =	get_encoder_layer_array(_A	,      "_attention_layer_norm/gamma"				)
		snake_case_						       =	get_encoder_layer_array(_A	,      "_attention_layer_norm/beta"				)

		# Intermediate
		snake_case_						       =	layer.intermediate

		snake_case_						       =	get_encoder_layer_array(_A	,      "_intermediate_dense/kernel"				)
		snake_case_						       =	get_encoder_layer_array(_A	,      "_intermediate_dense/bias"				)

		# Output
		snake_case_						       =	layer.output

		snake_case_						       =	get_encoder_layer_array(_A	,      "_output_dense/kernel"				)
		snake_case_						       =	get_encoder_layer_array(_A	,      "_output_dense/bias"				)

		snake_case_						       =	get_encoder_layer_array(_A	,      "_output_layer_norm/gamma"				)
		snake_case_						       =	get_encoder_layer_array(_A	,      "_output_layer_norm/beta"				)

	# Embeddings
	snake_case_						       =	get_encoder_array("_position_embedding_layer/embeddings"				)
	snake_case_						       =	get_encoder_array("_type_embedding_layer/embeddings"				)
	snake_case_						       =	get_encoder_array("_embedding_norm_layer/gamma"				)
	snake_case_						       =	get_encoder_array("_embedding_norm_layer/beta"				)

	# LM Head
	snake_case_						       =	model.cls.predictions.transform

	snake_case_						       =	get_masked_lm_array("dense/kernel"				)
	snake_case_						       =	get_masked_lm_array("dense/bias"				)

	snake_case_						       =	get_masked_lm_array("layer_norm/gamma"				)
	snake_case_						       =	get_masked_lm_array("layer_norm/beta"				)

	snake_case_						       =	get_masked_lm_array("embedding_table"				)

	# Pooling
	snake_case_						       =	BertPooler(config=_A				)
	snake_case_						       =	get_encoder_array("_pooler_layer/kernel"				)
	snake_case_						       =	get_encoder_array("_pooler_layer/bias"				)

	# Export final model
	model.save_pretrained(_A				)

	# Integration test - should load without any errors ;)
	snake_case_						       =	BertForMaskedLM.from_pretrained(_A				)
	print(new_model.eval()				)

	print("Model conversion was done sucessfully!"				)


if __name__ == "__main__":
		lowercase__							:	int											=		argparse.ArgumentParser()
		parser.add_argument(
		    "--tf_checkpoint_path", type=str, required=True, help="Path to the TensorFlow Token Dropping checkpoint path."
		)
		parser.add_argument(
		    "--bert_config_file",
		    type=str,
		    required=True,
		    help="The config json file corresponding to the BERT model. This specifies the model architecture.",
		)
		parser.add_argument(
		    "--pytorch_dump_path",
		    type=str,
		    required=True,
		    help="Path to the output PyTorch model.",
		)
		lowercase__							:	Optional[int]											=		parser.parse_args()
		convert_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)






import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
		from sagemaker import Session, TrainingJobAnalytics
		from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''			,  '''False'''     )     ) is not True			,  reason='''Skipping test because should only be run when releasing minor transformers version'''			,  )
@pytest.mark.usefixtures('''sm_env'''     )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 650, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_ddp.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.7, '''eval_loss''': 0.6},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf_dist.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.p3.16xlarge''',
            '''results''': {'''train_runtime''': 600, '''eval_accuracy''': 0.6, '''eval_loss''': 0.7},
        },
    ]     )
class 						UpperCAmelCase  (							unittest.TestCase     ):







						'''simple docstring'''





						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							if self.framework == "pytorch":
								subprocess.run(
								    f"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()			,      encoding="utf-8"			,      check=__lowercase			,      )
							assert hasattr(self			,      "env"		)






						def 			snake_case__		(		self				:	Dict			,      __lowercase				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	f"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
							# distributed data settings
							snake_case_						       =	{"smdistributed": {"dataparallel": {"enabled": True}}} if self.script != "run_ddp.py" else None

							# creates estimator
							return HuggingFace(
							    entry_point=self.script			,      source_dir=self.env.test_path			,      role=self.env.role			,      image_uri=self.env.image_uri			,      base_job_name=__lowercase			,      instance_count=__lowercase			,      instance_type=self.instance_type			,      debugger_hook_config=__lowercase			,      hyperparameters={**self.env.distributed_hyperparameters, "model_name_or_path": self.model_name_or_path}			,      metric_definitions=self.env.metric_definitions			,      distribution=__lowercase			,      py_version="py36"			,      )






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							TrainingJobAnalytics(__lowercase		).export_csv(f"{self.env.test_path}/{job_name}_metrics.csv"		)






						@parameterized.expand([(2,)]		)
						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							snake_case_						       =	self.create_estimator(__lowercase		)

							# run training
							estimator.fit()

							# result dataframe
							snake_case_						       =	TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

							# extract kpis
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]		)
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]		)
							# get train time from SageMaker job, this includes starting, preprocessing, stopping
							snake_case_						       =	(
							    Session().describe_training_job(estimator.latest_training_job.name		).get("TrainingTimeInSeconds"			,      99_99_99		)
							)

							# assert kpis
							assert train_runtime <= self.results["train_runtime"]
							assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy		)
							assert all(t <= self.results["eval_loss"] for t in eval_loss		)

							# dump tests result into json file to share in PR
							with open(f"{estimator.latest_training_job.name}.json"			,      "w"		) as outfile:
								json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}			,      __lowercase		)
