




import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE



class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= ''''''
	A    							= (
	    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
	)
	A    							= None  # compression type in fsspec. ex: "gzip"
	A    							= None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz





	def __init__(							self      ,     SCREAMING_SNAKE_CASE = ""      ,     SCREAMING_SNAKE_CASE = None      ,     SCREAMING_SNAKE_CASE = None      ,     **SCREAMING_SNAKE_CASE      )  ->					Dict:




				"""simple docstring"""



				super().__init__(self      ,     **lowerCamelCase__      )
				# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
				SCREAMING_SNAKE_CASE_								=						fsspec.open(
				    lowerCamelCase__      ,     mode='rb'      ,     protocol=lowerCamelCase__      ,     compression=self.compression      ,     client_kwargs={
				        'requote_redirect_url': False,  # see https://github.com/huggingface/datasets/pull/5459
				        'trust_env': True,  # Enable reading proxy env variables.
				        **(target_options or {}).pop('client_kwargs'      ,     {}      ),  # To avoid issues if it was already passed.
				    }      ,     **(target_options or {})      ,     )
				SCREAMING_SNAKE_CASE_								=						os.path.basename(self.file.path.split('::'      )[0]      )
				SCREAMING_SNAKE_CASE_								=						(
				    self.compressed_name[: self.compressed_name.rindex('.'      )]
				    if '.' in self.compressed_name
				    else self.compressed_name
				)
				SCREAMING_SNAKE_CASE_								=						None





	@classmethod
	def 						A_(							cls      ,     SCREAMING_SNAKE_CASE      )  ->					Optional[Any]:




				"""simple docstring"""



				return super()._strip_protocol(lowerCamelCase__      ).lstrip('/'      )





	def 						A_(							self      )  ->					List[str]:




				"""simple docstring"""



				if self.dir_cache is None:
							SCREAMING_SNAKE_CASE_								=						{**self.file.fs.info(self.file.path      ), 'name': self.uncompressed_name}
							SCREAMING_SNAKE_CASE_								=						{f['name']: f}





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					Tuple:




				"""simple docstring"""



				return self.file.open().read()





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = "rb"      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE=None      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					Optional[Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self._strip_protocol(lowerCamelCase__      )
				if mode != "rb":
							raise ValueError(f'Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''      )
				return self.file.open()




class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= '''bz2'''
	A    							= '''bz2'''
	A    							= '''.bz2'''




class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= '''gzip'''
	A    							= '''gzip'''
	A    							= '''.gz'''




class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= '''lz4'''
	A    							= '''lz4'''
	A    							= '''.lz4'''




class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= '''xz'''
	A    							= '''xz'''
	A    							= '''.xz'''




class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= '''zstd'''
	A    							= '''zstd'''
	A    							= '''.zst'''





	def __init__(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = "rb"      ,     SCREAMING_SNAKE_CASE = None      ,     SCREAMING_SNAKE_CASE = None      ,     SCREAMING_SNAKE_CASE = DEFAULT_BLOCK_SIZE      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					Dict:




				"""simple docstring"""



				super().__init__(
				    fo=lowerCamelCase__      ,     mode=lowerCamelCase__      ,     target_protocol=lowerCamelCase__      ,     target_options=lowerCamelCase__      ,     block_size=lowerCamelCase__      ,     **lowerCamelCase__      ,     )
				# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
				#
				# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
				#     out.close = close
				# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
				#
				# see https://github.com/intake/filesystem_spec/issues/725
				SCREAMING_SNAKE_CASE_								=						self.file.__enter__



				class       a_       :





					def __init__(							self      ,     SCREAMING_SNAKE_CASE      )  ->					str:




								"""simple docstring"""



								SCREAMING_SNAKE_CASE_								=						file_





					def __enter__(							self      )  ->					Optional[Any]:




								"""simple docstring"""



								self._file.__enter__()
								return self





					def __exit__(							self      ,     *SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      )  ->					int:




								"""simple docstring"""



								self._file.__exit__(*lowerCamelCase__      ,     **lowerCamelCase__      )





					def __iter__(							self      )  ->					List[str]:




								"""simple docstring"""



								return iter(self._file      )





					def 						A_(							self      )  ->					List[Any]:




								"""simple docstring"""



								return next(self._file      )





					def __getattr__(							self      ,     SCREAMING_SNAKE_CASE      )  ->					Union[str, Any]:




								"""simple docstring"""



								return getattr(self._file      ,     lowerCamelCase__      )

				def fixed_enter(*SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      ):
							return WrappedFile(_enter(*lowerCamelCase__      ,     **lowerCamelCase__      )      )

				SCREAMING_SNAKE_CASE_								=						fixed_enter






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A								=       data_utils.TransfoXLTokenizer
__A								=       data_utils.TransfoXLCorpus
__A								=       data_utils
__A								=       data_utils







def      lowerCamelCase_						(			UpperCamelCase__							: int					,	UpperCamelCase__							: str					,	UpperCamelCase__							: Optional[Any]					,	UpperCamelCase__							: Tuple )	->			Tuple:





				"""simple docstring"""


				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(UpperCamelCase__					,	'rb' ) as fp:
												__lowerCamelCase									=       pickle.load(UpperCamelCase__					,	encoding='latin1' )
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								__lowerCamelCase									=       pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
								print(F"""Save vocabulary to {pytorch_vocab_dump_path}""" )
								__lowerCamelCase									=       corpus.vocab.__dict__
								torch.save(UpperCamelCase__					,	UpperCamelCase__ )

								__lowerCamelCase									=       corpus.__dict__
								corpus_dict_no_vocab.pop('vocab'					,	UpperCamelCase__ )
								__lowerCamelCase									=       pytorch_dump_folder_path + '/' + CORPUS_NAME
								print(F"""Save dataset to {pytorch_dataset_dump_path}""" )
								torch.save(UpperCamelCase__					,	UpperCamelCase__ )

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								__lowerCamelCase									=       os.path.abspath(UpperCamelCase__ )
								__lowerCamelCase									=       os.path.abspath(UpperCamelCase__ )

								print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.""" )
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												__lowerCamelCase									=       TransfoXLConfig()
								else:
												__lowerCamelCase									=       TransfoXLConfig.from_json_file(UpperCamelCase__ )
								print(F"""Building PyTorch model from configuration: {config}""" )
								__lowerCamelCase									=       TransfoXLLMHeadModel(UpperCamelCase__ )

								__lowerCamelCase									=       load_tf_weights_in_transfo_xl(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )
								# Save pytorch-model
								__lowerCamelCase									=       os.path.join(UpperCamelCase__					,	UpperCamelCase__ )
								__lowerCamelCase									=       os.path.join(UpperCamelCase__					,	UpperCamelCase__ )
								print(F"""Save PyTorch model to {os.path.abspath(UpperCamelCase__ )}""" )
								torch.save(model.state_dict()					,	UpperCamelCase__ )
								print(F"""Save configuration file to {os.path.abspath(UpperCamelCase__ )}""" )
								with open(UpperCamelCase__					,	'w'					,	encoding='utf-8' ) as f:
												f.write(config.to_json_string() )


if __name__ == "__main__":
					__A								=       argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A								=       parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)


