



"""simple docstring"""


import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A   :							Optional[int]										=   data_utils.TransfoXLTokenizer
__A   :							Any										=   data_utils.TransfoXLCorpus
__A   :							str										=   data_utils
__A   :							List[Any]										=   data_utils







def  lowercase	(      _SCREAMING_SNAKE_CASE   :						int					,     _SCREAMING_SNAKE_CASE   :						Union[str, Any]					,     _SCREAMING_SNAKE_CASE   :						List[str]					,     _SCREAMING_SNAKE_CASE   :						Optional[int]				):






				'''simple docstring'''



				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(_SCREAMING_SNAKE_CASE					,     '''rb'''				) as fp:
												_UpperCAmelCase					=    pickle.load(_SCREAMING_SNAKE_CASE					,     encoding='''latin1'''				)
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								_UpperCAmelCase					=    pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
								print(f'Save vocabulary to {pytorch_vocab_dump_path}'				)
								_UpperCAmelCase					=    corpus.vocab.__dict__
								torch.save(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)

								_UpperCAmelCase					=    corpus.__dict__
								corpus_dict_no_vocab.pop('''vocab'''					,     _SCREAMING_SNAKE_CASE				)
								_UpperCAmelCase					=    pytorch_dump_folder_path + '''/''' + CORPUS_NAME
								print(f'Save dataset to {pytorch_dataset_dump_path}'				)
								torch.save(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								_UpperCAmelCase					=    os.path.abspath(_SCREAMING_SNAKE_CASE				)
								_UpperCAmelCase					=    os.path.abspath(_SCREAMING_SNAKE_CASE				)

								print(f'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'				)
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												_UpperCAmelCase					=    TransfoXLConfig()
								else:
												_UpperCAmelCase					=    TransfoXLConfig.from_json_file(_SCREAMING_SNAKE_CASE				)
								print(f'Building PyTorch model from configuration: {config}'				)
								_UpperCAmelCase					=    TransfoXLLMHeadModel(_SCREAMING_SNAKE_CASE				)

								_UpperCAmelCase					=    load_tf_weights_in_transfo_xl(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)
								# Save pytorch-model
								_UpperCAmelCase					=    os.path.join(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)
								_UpperCAmelCase					=    os.path.join(_SCREAMING_SNAKE_CASE					,     _SCREAMING_SNAKE_CASE				)
								print(f'Save PyTorch model to {os.path.abspath(_SCREAMING_SNAKE_CASE				)}'				)
								torch.save(model.state_dict()					,     _SCREAMING_SNAKE_CASE				)
								print(f'Save configuration file to {os.path.abspath(_SCREAMING_SNAKE_CASE				)}'				)
								with open(_SCREAMING_SNAKE_CASE					,     '''w'''					,     encoding='''utf-8'''				) as f:
												f.write(config.to_json_string()				)


if __name__ == "__main__":
					__A   :							Tuple										=   argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A   :							str										=   parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)








"""simple docstring"""


import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment
@require_huggingface_suite
class 			_a (					unittest.TestCase):
			"""simple docstring"""



			def  lowercase__						(		self     :    Tuple   )->int:
							_UpperCAmelCase					=    inspect.getfile(accelerate.test_utils   )
							_UpperCAmelCase					=    os.path.sep.join(
							    mod_file.split(os.path.sep   )[:-1] + ['''scripts''', '''external_deps''', '''test_metrics.py''']   )

							from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

							_UpperCAmelCase					=    test_metrics



			@require_cpu
			def  lowercase__						(		self     :    Any   )->int:
							debug_launcher(self.test_metrics.main				,   num_processes=1   )



			@require_cpu
			def  lowercase__						(		self     :    List[str]   )->List[str]:
							debug_launcher(self.test_metrics.main   )



			@require_single_gpu
			def  lowercase__						(		self     :    List[Any]   )->Dict:
							self.test_metrics.main()



			@require_multi_gpu
			def  lowercase__						(		self     :    str   )->int:
							print(F'Found {torch.cuda.device_count()} devices.'   )
							_UpperCAmelCase					=    ['''torchrun''', F'--nproc_per_node={torch.cuda.device_count()}', self.test_file_path]
							with patch_environment(omp_num_threads=1   ):
											execute_subprocess_async(__UpperCamelCase				,   env=os.environ.copy()   )



