



import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch

def      lowerCAmelCase_					(  _lowercase    :       List[Any])							->	Tuple:







			"""simple docstring"""
			a__       :						Any							=  os.path.join(args.tf_model_dir						,       """parameters.json""")
			a__       :						List[Any]							=  json.loads(open(_lowercase).read())
			if not params:
						raise ValueError(
						    F'''It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file.''')
			if not args.output.endswith(""".pt"""):
						a__       :						str							=  args.output + """.pt"""
			a__       :						List[Any]							=  OrderedDict()
			with tf.device("""/CPU:0"""):
						a__       :						List[str]							=  tf.train.load_checkpoint(args.tf_model_dir)
						a__       :						Tuple							=  reader.get_variable_to_shape_map()
						for key_name in shapes.keys():
									a__       :						Optional[Any]							=  reader.get_tensor(_lowercase).astype(np.floataa)
									if key_name.endswith("""/adam_m""") or key_name.endswith("""/adam_v"""):
												continue
									if key_name.startswith("""pasts/"""):
												if key_name.startswith("""pasts/mlp"""):
															a__       :						Optional[int]							=  int(key_name[9])
												elif key_name.startswith("""pasts/out"""):
															a__       :						List[str]							=  8
												a__       :						int							=  """model.sqout.%d.weight""" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
												a__       :						int							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
												a__       :						Optional[Any]							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/moe"""):
												a__       :						int							=  int(key_name[9:].split("""/""")[0])
												if key_name.endswith("""/switch_gating/kernel"""):
															a__       :						Optional[Any]							=  """model.blocks.%d.feed_forward.mlp.router.classifier.weight""" % player
															a__       :						Any							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Union[str, Any]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/softmlp/kernel"""):
															a__       :						str							=  """model.blocks.%d.feed_forward.soft_bypass_mlp.weight""" % player
															a__       :						int							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
															a__       :						List[str]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/wo/kernel""") or key_name.endswith("""/wi/kernel"""):
															a__       :						Union[str, Any]							=  key_name[-9:-7]
															for i in range(16):
																		a__       :						List[str]							=  """model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight""" % (player, i, nlayer)
																		a__       :						Dict							=  (
																		    vnp[i].transpose([1, 0]).copy()
																		)  # In Mesh-Tensorflow, it is one array, so it is divided
																		a__       :						int							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/mlp"""):
												a__       :						str							=  int(key_name[9:].split("""/""")[0])
												if key_name.endswith("""/p1/kernel"""):
															a__       :						Optional[Any]							=  """model.blocks.%d.feed_forward.mlp.wi.weight""" % player
															a__       :						Tuple							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Optional[Any]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/p1/bias"""):
															a__       :						Optional[int]							=  """model.blocks.%d.feed_forward.mlp.wi.bias""" % player
															a__       :						Optional[int]							=  vnp.copy()  # same because it is one dimensional
															a__       :						Optional[Any]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/p2/kernel"""):
															a__       :						List[Any]							=  """model.blocks.%d.feed_forward.mlp.wo.weight""" % player
															a__       :						Union[str, Any]							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Optional[int]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/p2/bias"""):
															a__       :						Tuple							=  """model.blocks.%d.feed_forward.mlp.wo.bias""" % player
															a__       :						Dict							=  vnp.copy()  # same because it is one dimensional
															a__       :						Any							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/ln"""):
												a__       :						int							=  int(key_name[8:].split("""/""")[0])
												if key_name.endswith("""/b"""):
															a__       :						Any							=  """model.blocks.%d.feed_forward.norm.bias""" % player
															a__       :						Any							=  vnp.copy()  # same because it is one dimensional
															a__       :						Dict							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/g"""):
															a__       :						Dict							=  """model.blocks.%d.feed_forward.norm.weight""" % player
															a__       :						List[str]							=  vnp.copy()  # same because it is one dimensional
															a__       :						str							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/att"""):
												a__       :						Optional[Any]							=  int(key_name[9:].split("""/""")[0])
												if key_name.endswith("""/qkv/kernel"""):
															a__       :						Optional[Any]							=  vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
															a__       :						str							=  state[:, 0, :, :]
															a__       :						Union[str, Any]							=  state[:, 1, :, :]
															a__       :						Union[str, Any]							=  state[:, 2, :, :]
															a__       :						Optional[int]							=  (
															    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]])
															    .transpose([1, 0])
															    .copy()
															)  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Optional[Any]							=  (
															    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]])
															    .transpose([1, 0])
															    .copy()
															)  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Dict							=  (
															    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]])
															    .transpose([1, 0])
															    .copy()
															)  # Mesh-Tensorflow is a diagonal matrix
															a__       :						Optional[Any]							=  """model.blocks.%d.self_attn.self_attn.q_proj.weight""" % player
															a__       :						int							=  torch.tensor(_lowercase)
															a__       :						Dict							=  """model.blocks.%d.self_attn.self_attn.k_proj.weight""" % player
															a__       :						List[str]							=  torch.tensor(_lowercase)
															a__       :						Any							=  """model.blocks.%d.self_attn.self_attn.v_proj.weight""" % player
															a__       :						List[str]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/o/kernel"""):
															a__       :						str							=  """model.blocks.%d.self_attn.self_attn.out_proj.weight""" % player
															a__       :						Optional[Any]							=  (
															    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]).transpose([1, 0]).copy()
															)  # Mesh-Tensorflow is a diagonal matrix
															a__       :						str							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/an"""):
												a__       :						str							=  int(key_name[8:].split("""/""")[0])
												if key_name.endswith("""/b"""):
															a__       :						Union[str, Any]							=  """model.blocks.%d.self_attn.norm.bias""" % player
															a__       :						int							=  vnp.copy()  # same because it is one dimensional
															a__       :						List[Any]							=  torch.tensor(_lowercase)
												elif key_name.endswith("""/g"""):
															a__       :						Optional[Any]							=  """model.blocks.%d.self_attn.norm.weight""" % player
															a__       :						List[str]							=  vnp.copy()  # same because it is one dimensional
															a__       :						Tuple							=  torch.tensor(_lowercase)
									elif (
									    key_name.startswith("""model/wte""")
									    or key_name.startswith("""model/wpe""")
									    or key_name.startswith("""model/ete""")
									):
												a__       :						List[Any]							=  {"""wte""": """embed_tokens""", """wpe""": """position_embeddings""", """ete""": """extra_position_embeddings"""}[
												    key_name[-3:]
												]
												a__       :						List[Any]							=  """model.%s.weight""" % nlayer
												a__       :						Dict							=  vnp.copy()  # same in embedded
												a__       :						Any							=  torch.tensor(_lowercase)
												if key_name.startswith("""model/wte"""):
															a__       :						int							=  """lm_head.weight"""
															a__       :						List[str]							=  vnp.copy()  # same in embedded
															a__       :						List[Any]							=  torch.tensor(_lowercase)
									elif key_name.startswith("""model/wob"""):
												a__       :						Any							=  """final_logits_bias"""
												a__       :						Optional[Any]							=  vnp.copy()  # same in embedded
												a__       :						List[str]							=  state.reshape((1, -1))
												a__       :						Any							=  torch.tensor(_lowercase)
									elif key_name == "model/dense/kernel":
												a__       :						str							=  """model.last_project.weight"""
												a__       :						Union[str, Any]							=  vnp.transpose([1, 0]).copy()  # Mesh-Tensorflow is a diagonal matrix
												a__       :						str							=  torch.tensor(_lowercase)
									elif key_name == "model/dense_1/bias":
												a__       :						Tuple							=  """model.last_project.bias"""
												a__       :						Union[str, Any]							=  vnp.copy()  # same because it is one dimensional
												a__       :						Union[str, Any]							=  torch.tensor(_lowercase)
			torch.save(_lowercase						,       args.output)


if __name__ == "__main__":
		_lowercase		:				int      		=argparse.ArgumentParser(
		    description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
		)
		parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
		parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
		_lowercase		:				Tuple      		=parser.parse_args()
		convert_tf_gptsan_to_pt(args)









import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
		import ctypes
		import msvcrt  # noqa

		class        snake_case__    (ctypes.Structure		):







								"""simple docstring"""




								__lowerCAmelCase					:Dict      =					[("size", ctypes.c_int), ("visible", ctypes.c_byte)]

def      lowerCAmelCase_					(  )							->	List[Any]:







			"""simple docstring"""
			if os.name == "nt":
						a__       :						int							=  CursorInfo()
						a__       :						Union[str, Any]							=  ctypes.windll.kernelaa.GetStdHandle(-11)
						ctypes.windll.kernelaa.GetConsoleCursorInfo(_lowercase						,       ctypes.byref(_lowercase))
						a__       :						List[str]							=  False
						ctypes.windll.kernelaa.SetConsoleCursorInfo(_lowercase						,       ctypes.byref(_lowercase))
			elif os.name == "posix":
						sys.stdout.write("""\033[?25l""")
						sys.stdout.flush()

def      lowerCAmelCase_					(  )							->	Optional[Any]:







			"""simple docstring"""
			if os.name == "nt":
						a__       :						List[Any]							=  CursorInfo()
						a__       :						Optional[int]							=  ctypes.windll.kernelaa.GetStdHandle(-11)
						ctypes.windll.kernelaa.GetConsoleCursorInfo(_lowercase						,       ctypes.byref(_lowercase))
						a__       :						Dict							=  True
						ctypes.windll.kernelaa.SetConsoleCursorInfo(_lowercase						,       ctypes.byref(_lowercase))
			elif os.name == "posix":
						sys.stdout.write("""\033[?25h""")
						sys.stdout.flush()





@contextmanager
def      lowerCAmelCase_					(  )							->	Any:







			"""simple docstring"""
			try:
						hide_cursor()
						yield
			finally:
						show_cursor()




