

'''simple docstring'''





import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
							import jax
							import jaxlib

UpperCamelCase_							=					get_logger()

UpperCamelCase_							=					None





class 		__SCREAMING_SNAKE_CASE       ( TensorFormatter[Mapping, 'jax.Array', Mapping]			):







				def __init__(					self    :		Union[str, Any]		,				UpperCAmelCase__    :		List[str]=None		,				UpperCAmelCase__    :		Tuple=None		,				**UpperCAmelCase__    :		Dict      ):
										'''simple docstring'''
										super().__init__(features=UpperCAmelCase__      )
										import jax
										from jaxlib.xla_client import Device

										if isinstance(UpperCAmelCase__		,				UpperCAmelCase__      ):
																raise ValueError(
																    F'''Expected {device} to be a `str` not {type(UpperCAmelCase__      )}, as `jaxlib.xla_extension.Device` '''
																    '''is not serializable neither with `pickle` nor with `dill`. Instead you can surround '''
																    '''the device with `str()` to get its string identifier that will be internally mapped '''
																    '''to the actual `jaxlib.xla_extension.Device`.'''      )
										lowercase   :       Dict								=device if isinstance(UpperCAmelCase__		,				UpperCAmelCase__      ) else str(jax.devices()[0]      )
										# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
										# with `pickle` nor with `dill`, so we need to use a global variable instead
										global DEVICE_MAPPING
										if DEVICE_MAPPING is None:
																lowercase   :       Optional[Any]								=self._map_devices_to_str()
										if self.device not in list(DEVICE_MAPPING.keys()      ):
																logger.warning(
																    F'''Device with string identifier {self.device} not listed among the available '''
																    F'''devices: {list(DEVICE_MAPPING.keys()      )}, so falling back to the default '''
																    F'''device: {str(jax.devices()[0]      )}.'''      )
																lowercase   :       List[Any]								=str(jax.devices()[0]      )
										lowercase   :       Optional[Any]								=jnp_array_kwargs







				@staticmethod
				def    lowerCamelCase_  (					):
										'''simple docstring'''
										import jax

										return {str(UpperCAmelCase__      ): device for device in jax.devices()}







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		List[Any]      ):
										'''simple docstring'''
										import jax
										import jax.numpy as jnp

										if isinstance(UpperCAmelCase__		,				UpperCAmelCase__      ) and column:
																if all(
																    isinstance(UpperCAmelCase__		,				jax.Array      ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column      ):
																						return jnp.stack(UpperCAmelCase__		,				axis=0      )
										return column







				def    lowerCamelCase_  (					self    :		int		,				UpperCAmelCase__    :		Union[str, Any]      ):
										'''simple docstring'''
										import jax
										import jax.numpy as jnp

										if isinstance(UpperCAmelCase__		,				(str, bytes, type(UpperCAmelCase__      ))      ):
																return value
										elif isinstance(UpperCAmelCase__		,				(np.character, np.ndarray)      ) and np.issubdtype(value.dtype		,				np.character      ):
																return value.tolist()

										lowercase   :       Dict								={}

										if isinstance(UpperCAmelCase__		,				(np.number, np.ndarray)      ) and np.issubdtype(value.dtype		,				np.integer      ):
																# the default int precision depends on the jax config
																# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
																if jax.config.jax_enable_xaa:
																						lowercase   :       Tuple								={'''dtype''': jnp.intaa}
																else:
																						lowercase   :       Optional[int]								={'''dtype''': jnp.intaa}
										elif isinstance(UpperCAmelCase__		,				(np.number, np.ndarray)      ) and np.issubdtype(value.dtype		,				np.floating      ):
																lowercase   :       Union[str, Any]								={'''dtype''': jnp.floataa}
										elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																import PIL.Image

																if isinstance(UpperCAmelCase__		,				PIL.Image.Image      ):
																						lowercase   :       int								=np.asarray(UpperCAmelCase__      )

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
										global DEVICE_MAPPING
										if DEVICE_MAPPING is None:
																lowercase   :       int								=self._map_devices_to_str()

										with jax.default_device(DEVICE_MAPPING[self.device]      ):
																# calling jnp.array on a np.ndarray does copy the data
																# see https://github.com/google/jax/issues/4486
																return jnp.array(UpperCAmelCase__		,				**{**default_dtype, **self.jnp_array_kwargs}      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				UpperCAmelCase__    :		Optional[int]      ):
										'''simple docstring'''
										import jax

										# support for torch, tf, jax etc.
										if config.TORCH_AVAILABLE and "torch" in sys.modules:
																import torch

																if isinstance(UpperCAmelCase__		,				torch.Tensor      ):
																						return self._tensorize(data_struct.detach().cpu().numpy()[()]      )
										if hasattr(UpperCAmelCase__		,				'''__array__'''      ) and not isinstance(UpperCAmelCase__		,				jax.Array      ):
																lowercase   :       Any								=data_struct.__array__()
										# support for nested types like struct of list of struct
										if isinstance(UpperCAmelCase__		,				np.ndarray      ):
																if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																						return self._consolidate([self.recursive_tensorize(UpperCAmelCase__      ) for substruct in data_struct]      )
										elif isinstance(UpperCAmelCase__		,				(list, tuple)      ):
																return self._consolidate([self.recursive_tensorize(UpperCAmelCase__      ) for substruct in data_struct]      )
										return self._tensorize(UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		int		,				UpperCAmelCase__    :		dict      ):
										'''simple docstring'''
										return map_nested(self._recursive_tensorize		,				UpperCAmelCase__		,				map_list=UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Optional[int]		,				UpperCAmelCase__    :		pa.Table      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=self.numpy_arrow_extractor().extract_row(UpperCAmelCase__      )
										lowercase   :       str								=self.python_features_decoder.decode_row(UpperCAmelCase__      )
										return self.recursive_tensorize(UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				UpperCAmelCase__    :		pa.Table      ):
										'''simple docstring'''
										lowercase   :       str								=self.numpy_arrow_extractor().extract_column(UpperCAmelCase__      )
										lowercase   :       Dict								=self.python_features_decoder.decode_column(UpperCAmelCase__		,				pa_table.column_names[0]      )
										lowercase   :       str								=self.recursive_tensorize(UpperCAmelCase__      )
										lowercase   :       Tuple								=self._consolidate(UpperCAmelCase__      )
										return column







				def    lowerCamelCase_  (					self    :		str		,				UpperCAmelCase__    :		pa.Table      ):
										'''simple docstring'''
										lowercase   :       int								=self.numpy_arrow_extractor().extract_batch(UpperCAmelCase__      )
										lowercase   :       Optional[int]								=self.python_features_decoder.decode_batch(UpperCAmelCase__      )
										lowercase   :       List[str]								=self.recursive_tensorize(UpperCAmelCase__      )
										for column_name in batch:
																lowercase   :       Optional[int]								=self._consolidate(batch[column_name]      )
										return batch








'''simple docstring'''





import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def 							_lowerCAmelCase (      __magic_name__	:    str	)							->     Union[str, Any]:
						lowercase   :       Union[str, Any]								=os.path.join(args.tf_model_dir      ,   '''parameters.json'''	)
						lowercase   :       List[str]								=json.loads(open(__magic_name__	).read()	)
						if not params:
												raise ValueError(
												    f'''It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file.'''	)
						if not args.output.endswith('''.pt'''	):
												lowercase   :       Tuple								=args.output + '''.pt'''
						lowercase   :       int								=OrderedDict()
						with tf.device('''/CPU:0'''	):
												lowercase   :       List[Any]								=tf.train.load_checkpoint(args.tf_model_dir	)
												lowercase   :       int								=reader.get_variable_to_shape_map()
												for key_name in shapes.keys():
																		lowercase   :       Any								=reader.get_tensor(__magic_name__	).astype(np.floataa	)
																		if key_name.endswith('''/adam_m'''	) or key_name.endswith('''/adam_v'''	):
																								continue
																		if key_name.startswith('''pasts/'''	):
																								if key_name.startswith('''pasts/mlp'''	):
																														lowercase   :       int								=int(key_name[9]	)
																								elif key_name.startswith('''pasts/out'''	):
																														lowercase   :       Union[str, Any]								=8
																								lowercase   :       Any								='''model.sqout.%d.weight''' % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																								lowercase   :       Dict								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/moe'''	):
																								lowercase   :       Union[str, Any]								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/switch_gating/kernel'''	):
																														lowercase   :       Dict								='''model.blocks.%d.feed_forward.mlp.router.classifier.weight''' % player
																														lowercase   :       Any								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/softmlp/kernel'''	):
																														lowercase   :       Optional[int]								='''model.blocks.%d.feed_forward.soft_bypass_mlp.weight''' % player
																														lowercase   :       int								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/wo/kernel'''	) or key_name.endswith('''/wi/kernel'''	):
																														lowercase   :       Union[str, Any]								=key_name[-9:-7]
																														for i in range(16	):
																																				lowercase   :       Dict								='''model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight''' % (player, i, nlayer)
																																				lowercase   :       Any								=(
																																				    vnp[i].transpose([1, 0]	).copy()
																																				)  # In Mesh-Tensorflow, it is one array, so it is divided
																																				lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/mlp'''	):
																								lowercase   :       Dict								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/p1/kernel'''	):
																														lowercase   :       Any								='''model.blocks.%d.feed_forward.mlp.wi.weight''' % player
																														lowercase   :       str								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Any								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p1/bias'''	):
																														lowercase   :       List[Any]								='''model.blocks.%d.feed_forward.mlp.wi.bias''' % player
																														lowercase   :       Union[str, Any]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p2/kernel'''	):
																														lowercase   :       int								='''model.blocks.%d.feed_forward.mlp.wo.weight''' % player
																														lowercase   :       Tuple								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       int								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p2/bias'''	):
																														lowercase   :       str								='''model.blocks.%d.feed_forward.mlp.wo.bias''' % player
																														lowercase   :       Optional[int]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/ln'''	):
																								lowercase   :       int								=int(key_name[8:].split('''/'''	)[0]	)
																								if key_name.endswith('''/b'''	):
																														lowercase   :       Any								='''model.blocks.%d.feed_forward.norm.bias''' % player
																														lowercase   :       Optional[int]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/g'''	):
																														lowercase   :       Optional[Any]								='''model.blocks.%d.feed_forward.norm.weight''' % player
																														lowercase   :       Any								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/att'''	):
																								lowercase   :       int								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/qkv/kernel'''	):
																														lowercase   :       Optional[int]								=vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																														lowercase   :       Dict								=state[:, 0, :, :]
																														lowercase   :       Tuple								=state[:, 1, :, :]
																														lowercase   :       List[Any]								=state[:, 2, :, :]
																														lowercase   :       Optional[int]								=(
																														    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[Any]								=(
																														    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[int]								=(
																														    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Dict								='''model.blocks.%d.self_attn.self_attn.q_proj.weight''' % player
																														lowercase   :       Dict								=torch.tensor(__magic_name__	)
																														lowercase   :       List[Any]								='''model.blocks.%d.self_attn.self_attn.k_proj.weight''' % player
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																														lowercase   :       Optional[Any]								='''model.blocks.%d.self_attn.self_attn.v_proj.weight''' % player
																														lowercase   :       Tuple								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/o/kernel'''	):
																														lowercase   :       Dict								='''model.blocks.%d.self_attn.self_attn.out_proj.weight''' % player
																														lowercase   :       List[Any]								=(
																														    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]	).transpose([1, 0]	).copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       str								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/an'''	):
																								lowercase   :       Optional[Any]								=int(key_name[8:].split('''/'''	)[0]	)
																								if key_name.endswith('''/b'''	):
																														lowercase   :       List[str]								='''model.blocks.%d.self_attn.norm.bias''' % player
																														lowercase   :       Union[str, Any]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/g'''	):
																														lowercase   :       Any								='''model.blocks.%d.self_attn.norm.weight''' % player
																														lowercase   :       Any								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																		elif (
																		    key_name.startswith('''model/wte'''	)
																		    or key_name.startswith('''model/wpe'''	)
																		    or key_name.startswith('''model/ete'''	)
																		):
																								lowercase   :       Any								={'''wte''': '''embed_tokens''', '''wpe''': '''position_embeddings''', '''ete''': '''extra_position_embeddings'''}[
																								    key_name[-3:]
																								]
																								lowercase   :       Optional[Any]								='''model.%s.weight''' % nlayer
																								lowercase   :       Optional[int]								=vnp.copy()  # same in embedded
																								lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																								if key_name.startswith('''model/wte'''	):
																														lowercase   :       Tuple								='''lm_head.weight'''
																														lowercase   :       str								=vnp.copy()  # same in embedded
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/wob'''	):
																								lowercase   :       List[str]								='''final_logits_bias'''
																								lowercase   :       Dict								=vnp.copy()  # same in embedded
																								lowercase   :       Tuple								=state.reshape((1, -1)	)
																								lowercase   :       Dict								=torch.tensor(__magic_name__	)
																		elif key_name == "model/dense/kernel":
																								lowercase   :       Dict								='''model.last_project.weight'''
																								lowercase   :       int								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																		elif key_name == "model/dense_1/bias":
																								lowercase   :       List[Any]								='''model.last_project.bias'''
																								lowercase   :       str								=vnp.copy()  # same because it is one dimensional
																								lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
						torch.save(__magic_name__      ,   args.output	)


if __name__ == "__main__":
							UpperCamelCase_							=					argparse.ArgumentParser(
							    description="""model converter.""", formatter_class=argparse.ArgumentDefaultsHelpFormatter
							)
							parser.add_argument("""--tf_model_dir""", metavar="""PATH""", type=str, required=True, help="""import model""")
							parser.add_argument("""--output""", metavar="""PATH""", type=str, required=True, help="""output model""")
							UpperCamelCase_							=					parser.parse_args()
							convert_tf_gptsan_to_pt(args)





