






import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel







def      snake_case		(							UpperCAmelCase	:       BertModel,						UpperCAmelCase	:       str,						UpperCAmelCase	:       str   ):
				A	 =      ('dense.weight', 'attention.self.query', 'attention.self.key', 'attention.self.value')

				A	 =      (
				    ('layer.', 'layer_'),
				    ('word_embeddings.weight', 'word_embeddings'),
				    ('position_embeddings.weight', 'position_embeddings'),
				    ('token_type_embeddings.weight', 'token_type_embeddings'),
				    ('.', '/'),
				    ('LayerNorm/weight', 'LayerNorm/gamma'),
				    ('LayerNorm/bias', 'LayerNorm/beta'),
				    ('weight', 'kernel'),
				)

				if not os.path.isdir(UpperCAmelCase   ):
								os.makedirs(UpperCAmelCase   )

				A	 =      model.state_dict()

				def to_tf_var_name(UpperCAmelCase	:       str   ):
								for patt, repl in iter(UpperCAmelCase   ):
												A	 =      name.replace(UpperCAmelCase,						UpperCAmelCase   )
								return f'bert/{name}'

				def create_tf_var(UpperCAmelCase	:       np.ndarray,						UpperCAmelCase	:       str,						UpperCAmelCase	:       tf.Session   ):
								A	 =      tf.dtypes.as_dtype(tensor.dtype   )
								A	 =      tf.get_variable(dtype=UpperCAmelCase,						shape=tensor.shape,						name=UpperCAmelCase,						initializer=tf.zeros_initializer()   )
								session.run(tf.variables_initializer([tf_var]   )   )
								session.run(UpperCAmelCase   )
								return tf_var

				tf.reset_default_graph()
				with tf.Session() as session:
								for var_name in state_dict:
												A	 =      to_tf_var_name(UpperCAmelCase   )
												A	 =      state_dict[var_name].numpy()
												if any(x in var_name for x in tensors_to_transpose   ):
																A	 =      torch_tensor.T
												A	 =      create_tf_var(tensor=UpperCAmelCase,						name=UpperCAmelCase,						session=UpperCAmelCase   )
												tf.keras.backend.set_value(UpperCAmelCase,						UpperCAmelCase   )
												A	 =      session.run(UpperCAmelCase   )
												print(f'Successfully created {tf_name}: {np.allclose(UpperCAmelCase,						UpperCAmelCase   )}'   )

								A	 =      tf.train.Saver(tf.trainable_variables()   )
								saver.save(UpperCAmelCase,						os.path.join(UpperCAmelCase,						model_name.replace('-',						'_'   ) + '.ckpt'   )   )







def      snake_case		(							UpperCAmelCase	:       str=None   ):
				A	 =      argparse.ArgumentParser()
				parser.add_argument('--model_name',						type=UpperCAmelCase,						required=UpperCAmelCase,						help='model name e.g. bert-base-uncased'   )
				parser.add_argument(
				    '--cache_dir',						type=UpperCAmelCase,						default=UpperCAmelCase,						required=UpperCAmelCase,						help='Directory containing pytorch model'   )
				parser.add_argument('--pytorch_model_path',						type=UpperCAmelCase,						required=UpperCAmelCase,						help='/path/to/<pytorch-model-name>.bin'   )
				parser.add_argument('--tf_cache_dir',						type=UpperCAmelCase,						required=UpperCAmelCase,						help='Directory in which to save tensorflow model'   )
				A	 =      parser.parse_args(UpperCAmelCase   )

				A	 =      BertModel.from_pretrained(
				    pretrained_model_name_or_path=args.model_name,						state_dict=torch.load(args.pytorch_model_path   ),						cache_dir=args.cache_dir,						)

				convert_pytorch_checkpoint_to_tf(model=UpperCAmelCase,						ckpt_dir=args.tf_cache_dir,						model_name=args.model_name   )


if __name__ == "__main__":
					main()













import argparse

import torch

from transformers import YosoConfig, YosoForMaskedLM







def      snake_case		(							UpperCAmelCase	:       List[Any]   ):
				if "model" in orig_key:
								A	 =      orig_key.replace('model.',						''   )
				if "norm1" in orig_key:
								A	 =      orig_key.replace('norm1',						'attention.output.LayerNorm'   )
				if "norm2" in orig_key:
								A	 =      orig_key.replace('norm2',						'output.LayerNorm'   )
				if "norm" in orig_key:
								A	 =      orig_key.replace('norm',						'LayerNorm'   )
				if "transformer" in orig_key:
								A	 =      orig_key.split('.'   )[0].split('_'   )[-1]
								A	 =      orig_key.replace(f'transformer_{layer_num}',						f'encoder.layer.{layer_num}'   )
				if "mha.attn" in orig_key:
								A	 =      orig_key.replace('mha.attn',						'attention.self'   )
				if "mha" in orig_key:
								A	 =      orig_key.replace('mha',						'attention'   )
				if "W_q" in orig_key:
								A	 =      orig_key.replace('W_q',						'self.query'   )
				if "W_k" in orig_key:
								A	 =      orig_key.replace('W_k',						'self.key'   )
				if "W_v" in orig_key:
								A	 =      orig_key.replace('W_v',						'self.value'   )
				if "ff1" in orig_key:
								A	 =      orig_key.replace('ff1',						'intermediate.dense'   )
				if "ff2" in orig_key:
								A	 =      orig_key.replace('ff2',						'output.dense'   )
				if "ff" in orig_key:
								A	 =      orig_key.replace('ff',						'output.dense'   )
				if "mlm_class" in orig_key:
								A	 =      orig_key.replace('mlm.mlm_class',						'cls.predictions.decoder'   )
				if "mlm" in orig_key:
								A	 =      orig_key.replace('mlm',						'cls.predictions.transform'   )
				if "cls" not in orig_key:
								A	 =      'yoso.' + orig_key

				return orig_key







def      snake_case		(							UpperCAmelCase	:       Tuple,						UpperCAmelCase	:       str   ):
				for key in orig_state_dict.copy().keys():
								A	 =      orig_state_dict.pop(UpperCAmelCase   )

								if ("pooler" in key) or ("sen_class" in key):
												continue
								else:
												A	 =      val

				A	 =      orig_state_dict['cls.predictions.decoder.bias']
				A	 =      torch.arange(UpperCAmelCase   ).expand((1, -1)   ) + 2

				return orig_state_dict







def      snake_case		(							UpperCAmelCase	:       int,						UpperCAmelCase	:       Union[str, Any],						UpperCAmelCase	:       str   ):
				A	 =      torch.load(UpperCAmelCase,						map_location='cpu'   )['model_state_dict']
				A	 =      YosoConfig.from_json_file(UpperCAmelCase   )
				A	 =      YosoForMaskedLM(UpperCAmelCase   )

				A	 =      convert_checkpoint_helper(config.max_position_embeddings,						UpperCAmelCase   )

				print(model.load_state_dict(UpperCAmelCase   )   )
				model.eval()
				model.save_pretrained(UpperCAmelCase   )

				print(f'Checkpoint successfuly converted. Model saved at {pytorch_dump_path}'   )


if __name__ == "__main__":
					lowerCAmelCase_           = argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    '--pytorch_model_path', default=None, type=str, required=True, help='Path to YOSO pytorch checkpoint.'
					)
					parser.add_argument(
					    '--config_file',
					    default=None,
					    type=str,
					    required=True,
					    help='The json file for YOSO model config.',
					)
					parser.add_argument(
					    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
					)
					lowerCAmelCase_           = parser.parse_args()
					convert_yoso_checkpoint(args.pytorch_model_path, args.config_file, args.pytorch_dump_path)





