


'''simple docstring'''
import argparse
import collections

import numpy as np
import torch
from flax import traverse_util
from tax import checkpoints

from transformers import MTaConfig, UMTaEncoderModel, UMTaForConditionalGeneration
from transformers.utils import logging


logging.set_verbosity_info()


def     a   (    __a				,      __a				,      __a     )						->			Optional[Any]:



					'''simple docstring'''






					return params[f'''{prefix}/{prefix}/relpos_bias/rel_embedding'''][:, i, :]


def     a   (    __a				,      __a				,      __a				,      __a="attention"     )						->			int:



					'''simple docstring'''






					UpperCamelCase__     :List[str]							      =				np.ascontiguousarray(params[f'''{prefix}/{prefix}/{layer_name}/key/kernel'''][:, i, :, :]     )
					UpperCamelCase__     :Tuple							      =				k_tmp.reshape(k_tmp.shape[0]				,      k_tmp.shape[1] * k_tmp.shape[2]     )
					UpperCamelCase__     :Optional[int]							      =				np.ascontiguousarray(params[f'''{prefix}/{prefix}/{layer_name}/out/kernel'''][:, i, :, :]     )
					UpperCamelCase__     :Union[str, Any]							      =				o_tmp.reshape(o_tmp.shape[0] * o_tmp.shape[1]				,      o_tmp.shape[2]     )
					UpperCamelCase__     :Optional[int]							      =				np.ascontiguousarray(params[f'''{prefix}/{prefix}/{layer_name}/query/kernel'''][:, i, :, :]     )
					UpperCamelCase__     :List[Any]							      =				q_tmp.reshape(q_tmp.shape[0]				,      q_tmp.shape[1] * q_tmp.shape[2]     )
					UpperCamelCase__     :List[Any]							      =				np.ascontiguousarray(params[f'''{prefix}/{prefix}/{layer_name}/value/kernel'''][:, i, :, :]     )
					UpperCamelCase__     :List[Any]							      =				v_tmp.reshape(v_tmp.shape[0]				,      v_tmp.shape[1] * v_tmp.shape[2]     )
					return k, o, q, v


def     a   (    __a				,      __a				,      __a				,      __a=False     )						->			List[str]:



					'''simple docstring'''






					if split_mlp_wi:
										UpperCamelCase__     :Any							      =				params[f'''{prefix}/{prefix}/mlp/wi_0/kernel'''][:, i, :]
										UpperCamelCase__     :Optional[int]							      =				params[f'''{prefix}/{prefix}/mlp/wi_1/kernel'''][:, i, :]
										UpperCamelCase__     :Any							      =				(wi_a, wi_a)
					else:
										UpperCamelCase__     :Optional[Any]							      =				params[f'''{prefix}/{prefix}/mlp/wi/kernel'''][:, i, :]

					UpperCamelCase__     :List[Any]							      =				params[f'''{prefix}/{prefix}/mlp/wo/kernel'''][:, i, :]
					return wi, wo


def     a   (    __a				,      __a				,      __a				,      __a     )						->			Any:



					'''simple docstring'''






					return params[f'''{prefix}/{prefix}/{layer_name}/scale'''][:, i]


def     a   (    __a				,      *, __a				,      __a				,      __a = False     )						->			List[str]:



					'''simple docstring'''






					UpperCamelCase__     :Tuple							      =				traverse_util.flatten_dict(variables['''target''']     )
					UpperCamelCase__     :int							      =				{'''/'''.join(__a     ): v for k, v in old.items()}

					# v1.1 models have a gated GeLU with wi_0 and wi_1 instead of wi
					UpperCamelCase__     :Dict							      =				'''encoder/encoder/mlp/wi_0/kernel''' in old
					print('''Split MLP:'''				,      __a     )

					UpperCamelCase__     :Union[str, Any]							      =				collections.OrderedDict()

					# Shared embeddings.
					UpperCamelCase__     :Union[str, Any]							      =				old['''token_embedder/embedding''']

					# Encoder.
					for i in range(__a     ):
										# Block i, layer 0 (Self Attention).
										UpperCamelCase__     :Union[str, Any]							      =				tax_layer_norm_lookup(__a				,      __a				,      '''encoder'''				,      '''pre_attention_layer_norm'''     )
										UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :Any							      =				tax_attention_lookup(__a				,      __a				,      '''encoder'''				,      '''attention'''     )
										UpperCamelCase__     :Any							      =				layer_norm
										UpperCamelCase__     :Any							      =				k.T
										UpperCamelCase__     :Union[str, Any]							      =				o.T
										UpperCamelCase__     :Tuple							      =				q.T
										UpperCamelCase__     :Optional[Any]							      =				v.T

										# Block i, layer 1 (MLP).
										UpperCamelCase__     :Optional[Any]							      =				tax_layer_norm_lookup(__a				,      __a				,      '''encoder'''				,      '''pre_mlp_layer_norm'''     )
										UpperCamelCase__      ,       UpperCamelCase__     :str							      =				tax_mlp_lookup(__a				,      __a				,      '''encoder'''				,      __a     )
										UpperCamelCase__     :Any							      =				layer_norm
										if split_mlp_wi:
															UpperCamelCase__     :int							      =				wi[0].T
															UpperCamelCase__     :List[Any]							      =				wi[1].T
										else:
															UpperCamelCase__     :List[str]							      =				wi.T
										UpperCamelCase__     :Union[str, Any]							      =				wo.T
										if scalable_attention:
															# convert the rel_embedding of each layer
															UpperCamelCase__     :Tuple							      =				tax_relpos_bias_lookup(
															    __a				,      __a				,      '''encoder'''     ).T

					UpperCamelCase__     :List[Any]							      =				old['''encoder/encoder_norm/scale''']

					if not scalable_attention:
										UpperCamelCase__     :Tuple							      =				tax_relpos_bias_lookup(
										    __a				,      0				,      '''encoder'''     ).T
										UpperCamelCase__     :Tuple							      =				tax_relpos_bias_lookup(
										    __a				,      0				,      '''decoder'''     ).T

					if not is_encoder_only:
										# Decoder.
										for i in range(__a     ):
															# Block i, layer 0 (Self Attention).
															UpperCamelCase__     :List[Any]							      =				tax_layer_norm_lookup(__a				,      __a				,      '''decoder'''				,      '''pre_self_attention_layer_norm'''     )
															UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :List[str]							      =				tax_attention_lookup(__a				,      __a				,      '''decoder'''				,      '''self_attention'''     )
															UpperCamelCase__     :Optional[Any]							      =				layer_norm
															UpperCamelCase__     :List[str]							      =				k.T
															UpperCamelCase__     :str							      =				o.T
															UpperCamelCase__     :Union[str, Any]							      =				q.T
															UpperCamelCase__     :List[Any]							      =				v.T

															# Block i, layer 1 (Cross Attention).
															UpperCamelCase__     :List[Any]							      =				tax_layer_norm_lookup(__a				,      __a				,      '''decoder'''				,      '''pre_cross_attention_layer_norm'''     )
															UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__      ,       UpperCamelCase__     :str							      =				tax_attention_lookup(__a				,      __a				,      '''decoder'''				,      '''encoder_decoder_attention'''     )
															UpperCamelCase__     :Dict							      =				layer_norm
															UpperCamelCase__     :Union[str, Any]							      =				k.T
															UpperCamelCase__     :Tuple							      =				o.T
															UpperCamelCase__     :Union[str, Any]							      =				q.T
															UpperCamelCase__     :Any							      =				v.T

															# Block i, layer 2 (MLP).
															UpperCamelCase__     :int							      =				tax_layer_norm_lookup(__a				,      __a				,      '''decoder'''				,      '''pre_mlp_layer_norm'''     )
															UpperCamelCase__      ,       UpperCamelCase__     :Union[str, Any]							      =				tax_mlp_lookup(__a				,      __a				,      '''decoder'''				,      __a     )
															UpperCamelCase__     :List[Any]							      =				layer_norm
															if split_mlp_wi:
																				UpperCamelCase__     :Optional[int]							      =				wi[0].T
																				UpperCamelCase__     :Dict							      =				wi[1].T
															else:
																				UpperCamelCase__     :List[str]							      =				wi.T
															UpperCamelCase__     :List[Any]							      =				wo.T

															if scalable_attention:
																				# convert the rel_embedding of each layer
																				UpperCamelCase__     :int							      =				tax_relpos_bias_lookup(__a				,      __a				,      '''decoder'''     ).T

										UpperCamelCase__     :Union[str, Any]							      =				old['''decoder/decoder_norm/scale''']

										# LM Head (only in v1.1 checkpoints, in v1.0 embeddings are used instead)
										if "decoder/logits_dense/kernel" in old:
															UpperCamelCase__     :int							      =				old['''decoder/logits_dense/kernel'''].T

					return new


def     a   (    __a				,      __a     )						->			Tuple:



					'''simple docstring'''






					UpperCamelCase__     :int							      =				collections.OrderedDict([(k, torch.from_numpy(v.copy()     )) for (k, v) in converted_params.items()]     )

					# Add what is missing.
					if "encoder.embed_tokens.weight" not in state_dict:
										UpperCamelCase__     :Optional[Any]							      =				state_dict['''shared.weight''']

					if not is_encoder_only:
										if "decoder.embed_tokens.weight" not in state_dict:
															UpperCamelCase__     :List[Any]							      =				state_dict['''shared.weight''']

										if "lm_head.weight" not in state_dict:  # For old 1.0 models.
															print('''Using shared word embeddings as lm_head.'''     )
															UpperCamelCase__     :Dict							      =				state_dict['''shared.weight''']

					return state_dict


def     a   (    __a				,      __a				,      __a				,      __a				,      __a     )						->			int:



					'''simple docstring'''






					UpperCamelCase__     :str							      =				checkpoints.load_tax_checkpoint(__a     )
					UpperCamelCase__     :List[str]							      =				convert_tax_to_pytorch(
					    __a				,      num_layers=config.num_layers				,      is_encoder_only=__a				,      scalable_attention=__a     )
					UpperCamelCase__     :List[Any]							      =				make_state_dict(__a				,      __a     )
					model.load_state_dict(__a				,      strict=__a     )


def     a   (    __a				,      __a				,      __a				,      __a = False				,      __a = False				,      )						->			Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__     :List[str]							      =				MTaConfig.from_json_file(__a     )
					print(f'''Building PyTorch model from configuration: {config}'''     )
					# Non-v1.1 checkpoints could also use T5Model, but this works for all.
					# The v1.0 checkpoints will simply have an LM head that is the word embeddings.
					if is_encoder_only:
										UpperCamelCase__     :Optional[int]							      =				UMTaEncoderModel(__a     )
					else:
										UpperCamelCase__     :List[Any]							      =				UMTaForConditionalGeneration(__a     )

					# Load weights from tf checkpoint
					load_tax_weights_in_ta(__a				,      __a				,      __a				,      __a				,      __a     )

					# Save pytorch-model
					print(f'''Save PyTorch model to {pytorch_dump_path}'''     )
					model.save_pretrained(__a     )

					# Verify that we can load the checkpoint.
					model.from_pretrained(__a     )
					print('''Done'''     )


if __name__ == "__main__":
		__snake_case							   =  argparse.ArgumentParser(description='''Converts a native T5X checkpoint into a PyTorch checkpoint.''')
		# Required parameters
		parser.add_argument(
		    '''--t5x_checkpoint_path''', default=None, type=str, required=True, help='''Path to the T5X checkpoint.'''
		)
		parser.add_argument(
		    '''--config_file''',
		    default=None,
		    type=str,
		    required=True,
		    help='''The config json file corresponding to the pre-trained T5 model.\nThis specifies the model architecture.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
		)
		parser.add_argument(
		    '''--is_encoder_only''', action='''store_true''', help='''Check if the model is encoder-decoder model''', default=False
		)
		parser.add_argument(
		    '''--scalable_attention''',
		    action='''store_true''',
		    help='''Whether the model uses scaled attention (umt5 model)''',
		    default=False,
		)
		__snake_case							   =  parser.parse_args()
		convert_tax_checkpoint_to_pytorch(
		    args.tax_checkpoint_path,
		    args.config_file,
		    args.pytorch_dump_path,
		    args.is_encoder_only,
		    args.scalable_attention,
		)



'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'transfo-xl'
		_a									=       ['mems']
		_a									=       {
		    'n_token': 'vocab_size',
		    'hidden_size': 'd_model',
		    'num_attention_heads': 'n_head',
		    'num_hidden_layers': 'n_layer',
		}




		def __init__(	self					,     UpperCamelCase_=267735					,     UpperCamelCase_=[20000, 40000, 200000]					,     UpperCamelCase_=1024					,     UpperCamelCase_=1024					,     UpperCamelCase_=16					,     UpperCamelCase_=64					,     UpperCamelCase_=4096					,     UpperCamelCase_=4					,     UpperCamelCase_=False					,     UpperCamelCase_=18					,     UpperCamelCase_=1600					,     UpperCamelCase_=1000					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=0					,     UpperCamelCase_=-1					,     UpperCamelCase_=True					,     UpperCamelCase_=0.1					,     UpperCamelCase_=0.0					,     UpperCamelCase_=True					,     UpperCamelCase_="normal"					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.02					,     UpperCamelCase_=1e-5					,     UpperCamelCase_=0					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				vocab_size
							UpperCamelCase__     :Optional[int]							      =				[]
							self.cutoffs.extend(UpperCamelCase_							)
							if proj_share_all_but_first:
												UpperCamelCase__     :Any							      =				[False] + [True] * len(self.cutoffs							)
							else:
												UpperCamelCase__     :Dict							      =				[False] + [False] * len(self.cutoffs							)
							UpperCamelCase__     :Union[str, Any]							      =				d_model
							UpperCamelCase__     :List[Any]							      =				d_embed
							UpperCamelCase__     :Any							      =				d_head
							UpperCamelCase__     :Tuple							      =				d_inner
							UpperCamelCase__     :int							      =				div_val
							UpperCamelCase__     :int							      =				pre_lnorm
							UpperCamelCase__     :Any							      =				n_layer
							UpperCamelCase__     :int							      =				n_head
							UpperCamelCase__     :Tuple							      =				mem_len
							UpperCamelCase__     :List[str]							      =				same_length
							UpperCamelCase__     :Any							      =				attn_type
							UpperCamelCase__     :List[str]							      =				clamp_len
							UpperCamelCase__     :Union[str, Any]							      =				sample_softmax
							UpperCamelCase__     :List[Any]							      =				adaptive
							UpperCamelCase__     :List[Any]							      =				dropout
							UpperCamelCase__     :Optional[Any]							      =				dropatt
							UpperCamelCase__     :str							      =				untie_r
							UpperCamelCase__     :List[Any]							      =				init
							UpperCamelCase__     :Optional[Any]							      =				init_range
							UpperCamelCase__     :int							      =				proj_init_std
							UpperCamelCase__     :List[str]							      =				init_std
							UpperCamelCase__     :Optional[int]							      =				layer_norm_epsilon
							super().__init__(eos_token_id=UpperCamelCase_					,     **UpperCamelCase_							)




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)
							return -1




		@max_position_embeddings.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)