
import torch
from torch import nn

class        snake_case__    (nn.Module		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase=1 ,       __lowercase=False  ) ->				int:
									"""simple docstring"""




									super().__init__()

									a__       :						Dict							=  n_token
									a__       :						Tuple							=  d_embed
									a__       :						Union[str, Any]							=  d_proj

									a__       :						List[Any]							=  cutoffs + [n_token]
									a__       :						Tuple							=  [0] + self.cutoffs
									a__       :						str							=  div_val

									a__       :						List[Any]							=  self.cutoffs[0]
									a__       :						Optional[int]							=  len(self.cutoffs  ) - 1
									a__       :						Optional[Any]							=  self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
												a__       :						Any							=  nn.Parameter(torch.zeros(self.n_clusters ,       self.d_embed  )  )
												a__       :						Optional[int]							=  nn.Parameter(torch.zeros(self.n_clusters  )  )

									a__       :						int							=  nn.ModuleList()
									a__       :						Optional[Any]							=  nn.ParameterList()

									if div_val == 1:
												for i in range(len(self.cutoffs  )  ):
															if d_proj != d_embed:
																		self.out_projs.append(nn.Parameter(torch.FloatTensor(a_ ,       a_  )  )  )
															else:
																		self.out_projs.append(a_  )

												self.out_layers.append(nn.Linear(a_ ,       a_  )  )
									else:
												for i in range(len(self.cutoffs  )  ):
															a__	,						a__       :						Optional[Any]							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
															a__       :						Tuple							=  d_embed // (div_val**i)

															self.out_projs.append(nn.Parameter(torch.FloatTensor(a_ ,       a_  )  )  )

															self.out_layers.append(nn.Linear(a_ ,       r_idx - l_idx  )  )

									a__       :						int							=  keep_order

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase  ) ->				Optional[Any]:
									"""simple docstring"""




									if proj is None:
												a__       :						Dict							=  nn.functional.linear(a_ ,       a_ ,       bias=a_  )
									else:
												# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
												a__       :						Optional[int]							=  nn.functional.linear(a_ ,       proj.t().contiguous()  )
												a__       :						Dict							=  nn.functional.linear(a_ ,       a_ ,       bias=a_  )
												# else:
												#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
												#     if bias is not None:
												#         logit = logit + bias

									return logit

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase=None ,       __lowercase=False  ) ->				Tuple:
									"""simple docstring"""




									if labels is not None:
												# Shift so that tokens < n predict n
												a__       :						str							=  hidden[..., :-1, :].contiguous()
												a__       :						List[str]							=  labels[..., 1:].contiguous()
												a__       :						Dict							=  hidden.view(-1 ,       hidden.size(-1  )  )
												a__       :						List[Any]							=  labels.view(-1  )
												if hidden.size(0  ) != labels.size(0  ):
															raise RuntimeError("""Input and labels should have the same size in the batch dimension."""  )
									else:
												a__       :						List[str]							=  hidden.view(-1 ,       hidden.size(-1  )  )

									if self.n_clusters == 0:
												a__       :						int							=  self._compute_logit(a_ ,       self.out_layers[0].weight ,       self.out_layers[0].bias ,       self.out_projs[0]  )
												if labels is not None:
															a__       :						str							=  labels != -1_0_0
															a__       :						List[Any]							=  torch.zeros_like(a_ ,       dtype=hidden.dtype ,       device=hidden.device  )
															a__       :						Union[str, Any]							=  (
															    -nn.functional.log_softmax(a_ ,       dim=-1  )[mask].gather(1 ,       labels[mask].unsqueeze(1  )  ).squeeze(1  )
															)
												else:
															a__       :						Union[str, Any]							=  nn.functional.log_softmax(a_ ,       dim=-1  )
									else:
												# construct weights and biases
												a__	,						a__       :						int							=  [], []
												for i in range(len(self.cutoffs  )  ):
															if self.div_val == 1:
																		a__	,						a__       :						Union[str, Any]							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
																		a__       :						Optional[Any]							=  self.out_layers[0].weight[l_idx:r_idx]
																		a__       :						int							=  self.out_layers[0].bias[l_idx:r_idx]
															else:
																		a__       :						Union[str, Any]							=  self.out_layers[i].weight
																		a__       :						Any							=  self.out_layers[i].bias

															if i == 0:
																		a__       :						List[str]							=  torch.cat([weight_i, self.cluster_weight] ,       dim=0  )
																		a__       :						Optional[Any]							=  torch.cat([bias_i, self.cluster_bias] ,       dim=0  )

															weights.append(a_  )
															biases.append(a_  )

												a__	,						a__	,						a__       :						List[str]							=  weights[0], biases[0], self.out_projs[0]

												a__       :						List[str]							=  self._compute_logit(a_ ,       a_ ,       a_ ,       a_  )
												a__       :						int							=  nn.functional.log_softmax(a_ ,       dim=1  )

												if labels is None:
															a__       :						List[Any]							=  hidden.new_empty((head_logit.size(0  ), self.n_token)  )
												else:
															a__       :						Tuple							=  torch.zeros_like(a_ ,       dtype=hidden.dtype ,       device=hidden.device  )

												a__       :						Optional[Any]							=  0
												a__       :						List[Any]							=  [0] + self.cutoffs
												for i in range(len(a_  ) - 1  ):
															a__	,						a__       :						Dict							=  cutoff_values[i], cutoff_values[i + 1]

															if labels is not None:
																		a__       :						Any							=  (labels >= l_idx) & (labels < r_idx)
																		a__       :						List[Any]							=  mask_i.nonzero().squeeze()

																		if indices_i.numel() == 0:
																					continue

																		a__       :						str							=  labels.index_select(0 ,       a_  ) - l_idx
																		a__       :						Optional[int]							=  head_logprob.index_select(0 ,       a_  )
																		a__       :						List[str]							=  hidden.index_select(0 ,       a_  )
															else:
																		a__       :						Optional[int]							=  hidden

															if i == 0:
																		if labels is not None:
																					a__       :						Tuple							=  head_logprob_i.gather(1 ,       target_i[:, None]  ).squeeze(1  )
																		else:
																					a__       :						int							=  head_logprob[:, : self.cutoffs[0]]
															else:
																		a__	,						a__	,						a__       :						str							=  weights[i], biases[i], self.out_projs[i]

																		a__       :						Optional[Any]							=  self._compute_logit(a_ ,       a_ ,       a_ ,       a_  )
																		a__       :						List[str]							=  nn.functional.log_softmax(a_ ,       dim=1  )
																		a__       :						Union[str, Any]							=  self.cutoffs[0] + i - 1  # No probability for the head cluster
																		if labels is not None:
																					a__       :						Optional[Any]							=  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																					    1 ,       target_i[:, None]  ).squeeze(1  )
																		else:
																					a__       :						List[str]							=  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																					a__       :						List[Any]							=  logprob_i

															if labels is not None:
																		if (hasattr(self ,       """keep_order"""  ) and self.keep_order) or keep_order:
																					out.index_copy_(0 ,       a_ ,       -logprob_i  )
																		else:
																					out[offset : offset + logprob_i.size(0  )].copy_(-logprob_i  )
																		offset += logprob_i.size(0  )

									return out

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				List[str]:
									"""simple docstring"""




									if self.n_clusters == 0:
												a__       :						Union[str, Any]							=  self._compute_logit(a_ ,       self.out_layers[0].weight ,       self.out_layers[0].bias ,       self.out_projs[0]  )
												return nn.functional.log_softmax(a_ ,       dim=-1  )
									else:
												# construct weights and biases
												a__	,						a__       :						Optional[Any]							=  [], []
												for i in range(len(self.cutoffs  )  ):
															if self.div_val == 1:
																		a__	,						a__       :						Optional[int]							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
																		a__       :						str							=  self.out_layers[0].weight[l_idx:r_idx]
																		a__       :						Tuple							=  self.out_layers[0].bias[l_idx:r_idx]
															else:
																		a__       :						Tuple							=  self.out_layers[i].weight
																		a__       :						int							=  self.out_layers[i].bias

															if i == 0:
																		a__       :						Dict							=  torch.cat([weight_i, self.cluster_weight] ,       dim=0  )
																		a__       :						Union[str, Any]							=  torch.cat([bias_i, self.cluster_bias] ,       dim=0  )

															weights.append(a_  )
															biases.append(a_  )

												a__	,						a__	,						a__       :						int							=  weights[0], biases[0], self.out_projs[0]
												a__       :						Optional[Any]							=  self._compute_logit(a_ ,       a_ ,       a_ ,       a_  )

												a__       :						Optional[int]							=  hidden.new_empty((head_logit.size(0  ), self.n_token)  )
												a__       :						str							=  nn.functional.log_softmax(a_ ,       dim=1  )

												a__       :						Dict							=  [0] + self.cutoffs
												for i in range(len(a_  ) - 1  ):
															a__	,						a__       :						str							=  cutoff_values[i], cutoff_values[i + 1]

															if i == 0:
																		a__       :						Union[str, Any]							=  head_logprob[:, : self.cutoffs[0]]
															else:
																		a__	,						a__	,						a__       :						Union[str, Any]							=  weights[i], biases[i], self.out_projs[i]

																		a__       :						Optional[Any]							=  self._compute_logit(a_ ,       a_ ,       a_ ,       a_  )
																		a__       :						Union[str, Any]							=  nn.functional.log_softmax(a_ ,       dim=1  )

																		a__       :						Any							=  head_logprob[:, -i] + tail_logprob_i
																		a__       :						int							=  logprob_i

												return out









import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowercase		:				Union[str, Any]      		=data_utils.TransfoXLTokenizer
_lowercase		:				Tuple      		=data_utils.TransfoXLCorpus
_lowercase		:				Optional[int]      		=data_utils
_lowercase		:				int      		=data_utils

def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       List[Any]						,       _lowercase    :       Optional[Any]						,       _lowercase    :       Optional[Any])							->	List[Any]:







			"""simple docstring"""
			if transfo_xl_dataset_file:
						# Convert a pre-processed corpus (see original TensorFlow repo)
						with open(_lowercase						,       """rb""") as fp:
									a__       :						Optional[Any]							=  pickle.load(_lowercase						,       encoding="""latin1""")
						# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
						a__       :						Dict							=  pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
						print(F'''Save vocabulary to {pytorch_vocab_dump_path}''')
						a__       :						List[str]							=  corpus.vocab.__dict__
						torch.save(_lowercase						,       _lowercase)

						a__       :						Optional[int]							=  corpus.__dict__
						corpus_dict_no_vocab.pop("""vocab"""						,       _lowercase)
						a__       :						Tuple							=  pytorch_dump_folder_path + """/""" + CORPUS_NAME
						print(F'''Save dataset to {pytorch_dataset_dump_path}''')
						torch.save(_lowercase						,       _lowercase)

			if tf_checkpoint_path:
						# Convert a pre-trained TensorFlow model
						a__       :						Optional[Any]							=  os.path.abspath(_lowercase)
						a__       :						str							=  os.path.abspath(_lowercase)

						print(F'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.''')
						# Initialise PyTorch model
						if transfo_xl_config_file == "":
									a__       :						List[str]							=  TransfoXLConfig()
						else:
									a__       :						Any							=  TransfoXLConfig.from_json_file(_lowercase)
						print(F'''Building PyTorch model from configuration: {config}''')
						a__       :						Any							=  TransfoXLLMHeadModel(_lowercase)

						a__       :						Optional[Any]							=  load_tf_weights_in_transfo_xl(_lowercase						,       _lowercase						,       _lowercase)
						# Save pytorch-model
						a__       :						Tuple							=  os.path.join(_lowercase						,       _lowercase)
						a__       :						Optional[int]							=  os.path.join(_lowercase						,       _lowercase)
						print(F'''Save PyTorch model to {os.path.abspath(_lowercase)}''')
						torch.save(model.state_dict()						,       _lowercase)
						print(F'''Save configuration file to {os.path.abspath(_lowercase)}''')
						with open(_lowercase						,       """w"""						,       encoding="""utf-8""") as f:
									f.write(config.to_json_string())


if __name__ == "__main__":
		_lowercase		:				Tuple      		=argparse.ArgumentParser()
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path to the folder to store the PyTorch model or dataset/vocab.",
		)
		parser.add_argument(
		    "--tf_checkpoint_path",
		    default="",
		    type=str,
		    help="An optional path to a TensorFlow checkpoint path to be converted.",
		)
		parser.add_argument(
		    "--transfo_xl_config_file",
		    default="",
		    type=str,
		    help=(
		        "An optional config json file corresponding to the pre-trained BERT model. \n"
		        "This specifies the model architecture."
		    ),
		)
		parser.add_argument(
		    "--transfo_xl_dataset_file",
		    default="",
		    type=str,
		    help="An optional dataset file to be converted in a vocabulary.",
		)
		_lowercase		:				Dict      		=parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)




