






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__snake_case    :    List[Any]         =     data_utils.TransfoXLTokenizer
__snake_case    :    Dict         =     data_utils.TransfoXLCorpus
__snake_case    :    Optional[int]         =     data_utils
__snake_case    :    Union[str, Any]         =     data_utils







def 	_UpperCamelCase   (    UpperCamelCase_       :					Union[str, Any]    ,       UpperCamelCase_       :					List[str]    ,       UpperCamelCase_       :					str    ,       UpperCamelCase_       :					int		)  ->   Tuple:

						"""simple docstring"""



						if transfo_xl_dataset_file:
												# Convert a pre-processed corpus (see original TensorFlow repo)
												with open(lowerCamelCase_    ,       'rb'		) as fp:
																		lowerCAmelCase__    						=						pickle.load(lowerCamelCase_    ,       encoding='latin1'		)
												# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
												lowerCAmelCase__    						=						pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
												print(F"Save vocabulary to {pytorch_vocab_dump_path}"		)
												lowerCAmelCase__    						=						corpus.vocab.__dict__
												torch.save(lowerCamelCase_    ,       lowerCamelCase_		)

												lowerCAmelCase__    						=						corpus.__dict__
												corpus_dict_no_vocab.pop('vocab'    ,       lowerCamelCase_		)
												lowerCAmelCase__    						=						pytorch_dump_folder_path + '/' + CORPUS_NAME
												print(F"Save dataset to {pytorch_dataset_dump_path}"		)
												torch.save(lowerCamelCase_    ,       lowerCamelCase_		)

						if tf_checkpoint_path:
												# Convert a pre-trained TensorFlow model
												lowerCAmelCase__    						=						os.path.abspath(lowerCamelCase_		)
												lowerCAmelCase__    						=						os.path.abspath(lowerCamelCase_		)

												print(F"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."		)
												# Initialise PyTorch model
												if transfo_xl_config_file == "":
																		lowerCAmelCase__    						=						TransfoXLConfig()
												else:
																		lowerCAmelCase__    						=						TransfoXLConfig.from_json_file(lowerCamelCase_		)
												print(F"Building PyTorch model from configuration: {config}"		)
												lowerCAmelCase__    						=						TransfoXLLMHeadModel(lowerCamelCase_		)

												lowerCAmelCase__    						=						load_tf_weights_in_transfo_xl(lowerCamelCase_    ,       lowerCamelCase_    ,       lowerCamelCase_		)
												# Save pytorch-model
												lowerCAmelCase__    						=						os.path.join(lowerCamelCase_    ,       lowerCamelCase_		)
												lowerCAmelCase__    						=						os.path.join(lowerCamelCase_    ,       lowerCamelCase_		)
												print(F"Save PyTorch model to {os.path.abspath(lowerCamelCase_		)}"		)
												torch.save(model.state_dict()    ,       lowerCamelCase_		)
												print(F"Save configuration file to {os.path.abspath(lowerCamelCase_		)}"		)
												with open(lowerCamelCase_    ,       'w'    ,       encoding='utf-8'		) as f:
																		f.write(config.to_json_string()		)


if __name__ == "__main__":
					__snake_case    :    Tuple         =     argparse.ArgumentParser()
					parser.add_argument(
					    """--pytorch_dump_folder_path""",
					    default=None,
					    type=str,
					    required=True,
					    help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
					)
					parser.add_argument(
					    """--tf_checkpoint_path""",
					    default="""""",
					    type=str,
					    help="""An optional path to a TensorFlow checkpoint path to be converted.""",
					)
					parser.add_argument(
					    """--transfo_xl_config_file""",
					    default="""""",
					    type=str,
					    help=(
					        """An optional config json file corresponding to the pre-trained BERT model. \n"""
					        """This specifies the model architecture."""
					    ),
					)
					parser.add_argument(
					    """--transfo_xl_dataset_file""",
					    default="""""",
					    type=str,
					    help="""An optional dataset file to be converted in a vocabulary.""",
					)
					__snake_case    :    Optional[int]         =     parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)













from collections import deque

from .hash_table import HashTable


class     __SCREAMING_SNAKE_CASE					(   __lowercase):






						def __init__(	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												super().__init__(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						deque([]   ) if self.values[key] is None else self.values[key]
												self.values[key].appendleft(_UpperCamelCase   )
												lowerCAmelCase__    						=						self.values[key]






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												return (
												    sum(self.charge_factor - len(_UpperCamelCase   ) for slot in self.values   )
												    / self.size_table
												    * self.charge_factor
												)







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase=None   ):






												"""simple docstring"""
												if not (
												    len(self.values[key]   ) == self.charge_factor and self.values.count(_UpperCamelCase   ) == 0
												):
																		return key
												return super()._collision_resolution(_UpperCamelCase     ,						_UpperCamelCase   )





