
"""simple docstring"""







import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__           =	get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 				A__ (			_lowerCamelCase ,  unittest.TestCase):
		A_		: Any            =			XLMRobertaTokenizer
		A_		: Optional[int]            =			XLMRobertaTokenizerFast
		A_		: Optional[Any]            =			True
		A_		: int            =			True

		def 							__lowerCamelCase							(   self				):
			super().setUp()

			# We have a SentencePiece fixture for testing
			__lowerCAmelCase    :		int		    =   XLMRobertaTokenizer(_SCREAMING_SNAKE_CASE ,	keep_accents=_SCREAMING_SNAKE_CASE				)
			tokenizer.save_pretrained(self.tmpdirname				)

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Dict		    =   '<pad>'
			__lowerCAmelCase    :		Tuple		    =   1

			self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE				) ,	_SCREAMING_SNAKE_CASE				)
			self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE				) ,	_SCREAMING_SNAKE_CASE				)

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		List[str]		    =   list(self.get_tokenizer().get_vocab().keys()				)

			self.assertEqual(vocab_keys[0] ,	'<s>'				)
			self.assertEqual(vocab_keys[1] ,	'<pad>'				)
			self.assertEqual(vocab_keys[-1] ,	'<mask>'				)
			self.assertEqual(len(_SCREAMING_SNAKE_CASE				) ,	10_02				)

		def 							__lowerCamelCase							(   self				):
			self.assertEqual(self.get_tokenizer().vocab_size ,	10_02				)

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		int		    =   XLMRobertaTokenizer(_SCREAMING_SNAKE_CASE ,	keep_accents=_SCREAMING_SNAKE_CASE				)

			__lowerCAmelCase    :		Optional[Any]		    =   tokenizer.tokenize('This is a test'				)
			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	['▁This', '▁is', '▁a', '▁t', 'est']				)

			self.assertListEqual(
			    tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				) ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]] ,	)

			__lowerCAmelCase    :		Dict		    =   tokenizer.tokenize('I was born in 92000, and this is falsé.'				)
			self.assertListEqual(
			    _SCREAMING_SNAKE_CASE ,	[
			        SPIECE_UNDERLINE + 'I',
			        SPIECE_UNDERLINE + 'was',
			        SPIECE_UNDERLINE + 'b',
			        'or',
			        'n',
			        SPIECE_UNDERLINE + 'in',
			        SPIECE_UNDERLINE + '',
			        '9',
			        '2',
			        '0',
			        '0',
			        '0',
			        ',',
			        SPIECE_UNDERLINE + 'and',
			        SPIECE_UNDERLINE + 'this',
			        SPIECE_UNDERLINE + 'is',
			        SPIECE_UNDERLINE + 'f',
			        'al',
			        's',
			        'é',
			        '.',
			    ] ,	)
			__lowerCAmelCase    :		Dict		    =   tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)
			self.assertListEqual(
			    _SCREAMING_SNAKE_CASE ,	[
			        value + tokenizer.fairseq_offset
			        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
			        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
			    ] ,	)

			__lowerCAmelCase    :		int		    =   tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
			self.assertListEqual(
			    _SCREAMING_SNAKE_CASE ,	[
			        SPIECE_UNDERLINE + 'I',
			        SPIECE_UNDERLINE + 'was',
			        SPIECE_UNDERLINE + 'b',
			        'or',
			        'n',
			        SPIECE_UNDERLINE + 'in',
			        SPIECE_UNDERLINE + '',
			        '<unk>',
			        '2',
			        '0',
			        '0',
			        '0',
			        ',',
			        SPIECE_UNDERLINE + 'and',
			        SPIECE_UNDERLINE + 'this',
			        SPIECE_UNDERLINE + 'is',
			        SPIECE_UNDERLINE + 'f',
			        'al',
			        's',
			        '<unk>',
			        '.',
			    ] ,	)

		def 							__lowerCamelCase							(   self				):
			if not self.test_slow_tokenizer:
				# as we don't have a slow version, we can't compare the outputs between slow and fast versions
				return

			__lowerCAmelCase    :		List[str]		    =   (self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"				):
					__lowerCAmelCase    :		str		    =   self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,	**_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		List[Any]		    =   self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,	**_SCREAMING_SNAKE_CASE				)

					__lowerCAmelCase    :		List[Any]		    =   tempfile.mkdtemp()

					__lowerCAmelCase    :		List[Any]		    =   tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		str		    =   tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE				)

					# Checks it save with the same files + the tokenizer.json file for the fast one
					self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files				)				)
					__lowerCAmelCase    :		int		    =   tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f				)
					self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)

					# Checks everything loads correctly in the same way
					__lowerCAmelCase    :		Any		    =   tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		int		    =   tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE				)

					# Check special tokens are set accordingly on Rust and Python
					for key in tokenizer_pp.special_tokens_map:
						self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)				)
						# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
						# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

					shutil.rmtree(_SCREAMING_SNAKE_CASE				)

					# Save tokenizer rust, legacy_format=True
					__lowerCAmelCase    :		Optional[Any]		    =   tempfile.mkdtemp()

					__lowerCAmelCase    :		Tuple		    =   tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,	legacy_format=_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		Optional[int]		    =   tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE				)

					# Checks it save with the same files
					self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)

					# Checks everything loads correctly in the same way
					__lowerCAmelCase    :		List[str]		    =   tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		Tuple		    =   tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE				)

					# Check special tokens are set accordingly on Rust and Python
					for key in tokenizer_pp.special_tokens_map:
						self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)				)

					shutil.rmtree(_SCREAMING_SNAKE_CASE				)

					# Save tokenizer rust, legacy_format=False
					__lowerCAmelCase    :		int		    =   tempfile.mkdtemp()

					__lowerCAmelCase    :		List[str]		    =   tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,	legacy_format=_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		Dict		    =   tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE				)

					# Checks it saved the tokenizer.json file
					self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files				)				)

					# Checks everything loads correctly in the same way
					__lowerCAmelCase    :		List[Any]		    =   tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE				)
					__lowerCAmelCase    :		Any		    =   tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE				)

					# Check special tokens are set accordingly on Rust and Python
					for key in tokenizer_pp.special_tokens_map:
						self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)				)

					shutil.rmtree(_SCREAMING_SNAKE_CASE				)

		@cached_property
		def 							__lowerCamelCase							(   self				):
			return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'				)

		def 							__lowerCamelCase							(   self				):
			with tempfile.NamedTemporaryFile() as f:
				shutil.copyfile(_SCREAMING_SNAKE_CASE ,	f.name				)
				__lowerCAmelCase    :		int		    =   XLMRobertaTokenizer(f.name ,	keep_accents=_SCREAMING_SNAKE_CASE				)
				__lowerCAmelCase    :		Optional[Any]		    =   pickle.dumps(_SCREAMING_SNAKE_CASE				)
			pickle.loads(_SCREAMING_SNAKE_CASE				)

		def 							__lowerCamelCase							(   self				):
			if not self.test_rust_tokenizer:
				return

			__lowerCAmelCase    :		int		    =   self.get_tokenizer()
			__lowerCAmelCase    :		List[Any]		    =   self.get_rust_tokenizer()

			__lowerCAmelCase    :		Optional[Any]		    =   'I was born in 92000, and this is falsé.'

			__lowerCAmelCase    :		Any		    =   tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
			__lowerCAmelCase    :		Tuple		    =   rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)

			__lowerCAmelCase    :		Dict		    =   tokenizer.encode(_SCREAMING_SNAKE_CASE ,	add_special_tokens=_SCREAMING_SNAKE_CASE				)
			__lowerCAmelCase    :		Optional[int]		    =   rust_tokenizer.encode(_SCREAMING_SNAKE_CASE ,	add_special_tokens=_SCREAMING_SNAKE_CASE				)
			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)

			__lowerCAmelCase    :		List[Any]		    =   self.get_rust_tokenizer()
			__lowerCAmelCase    :		List[Any]		    =   tokenizer.encode(_SCREAMING_SNAKE_CASE				)
			__lowerCAmelCase    :		Optional[int]		    =   rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE				)

		@slow
		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Tuple		    =   'Hello World!'
			__lowerCAmelCase    :		Union[str, Any]		    =   [0, 3_53_78, 66_61, 38, 2]
			# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
			# xlmr.eval()
			# xlmr.encode(symbols)

			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE				)				)

		@slow
		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		str		    =   (
			    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
			    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
			)
			__lowerCAmelCase    :		List[Any]		    =   [
			    0,
			    32_93,
			    83,
			    10,
			    45_52,
			    49_89,
			    79_86,
			    6_78,
			    10,
			    59_15,
			    1_11,
			    17_94_59,
			    12_48_50,
			    4,
			    60_44,
			    2_37,
			    12,
			    6,
			    5,
			    6,
			    4,
			    67_80,
			    7_05,
			    15,
			    13_88,
			    44,
			    3_78,
			    1_01_14,
			    7_11,
			    1_52,
			    20,
			    6,
			    5,
			    2_23_76,
			    6_42,
			    12_21,
			    1_51_90,
			    3_41_53,
			    4_50,
			    56_08,
			    9_59,
			    11_19,
			    5_77_02,
			    1_36,
			    1_86,
			    47,
			    10_98,
			    2_93_67,
			    47,
			    # 4426, # What fairseq tokenizes from "<unk>": "_<"
			    # 3678, # What fairseq tokenizes from "<unk>": "unk"
			    # 2740, # What fairseq tokenizes from "<unk>": ">"
			    3,  # What we tokenize from "<unk>": "<unk>"
			    6,  # Residue from the tokenization: an extra sentencepiece underline
			    4,
			    60_44,
			    2_37,
			    62_84,
			    5_09_01,
			    5_28,
			    31,
			    90,
			    34,
			    9_27,
			    2,
			]
			# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
			# xlmr.eval()
			# xlmr.encode(symbols)

			self.assertListEqual(_SCREAMING_SNAKE_CASE ,	self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE				)				)







		@slow
		def 							__lowerCamelCase							(   self				):
			# fmt: off
			__lowerCAmelCase    :		Union[str, Any]		    =   {'input_ids': [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
			# fmt: on

			self.tokenizer_integration_test_util(
			    expected_encoding=_SCREAMING_SNAKE_CASE ,	model_name='xlm-roberta-base' ,	revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3' ,	)

"""simple docstring"""







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCamelCase__           =	data_utils.TransfoXLTokenizer
lowerCamelCase__           =	data_utils.TransfoXLCorpus
lowerCamelCase__           =	data_utils
lowerCamelCase__           =	data_utils
def 					__lowerCAmelCase       (_UpperCamelCase ,						_UpperCamelCase ,						_UpperCamelCase ,						_UpperCamelCase	):
	if transfo_xl_dataset_file:
		# Convert a pre-processed corpus (see original TensorFlow repo)
		with open(_UpperCamelCase ,						'rb'	) as fp:
			__lowerCAmelCase    :		int		    =   pickle.load(_UpperCamelCase ,						encoding='latin1'	)
		# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
		__lowerCAmelCase    :		Any		    =   pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
		print(F"Save vocabulary to {pytorch_vocab_dump_path}"	)
		__lowerCAmelCase    :		Optional[Any]		    =   corpus.vocab.__dict__
		torch.save(_UpperCamelCase ,						_UpperCamelCase	)

		__lowerCAmelCase    :		Dict		    =   corpus.__dict__
		corpus_dict_no_vocab.pop('vocab' ,						_UpperCamelCase	)
		__lowerCAmelCase    :		List[str]		    =   pytorch_dump_folder_path + '/' + CORPUS_NAME
		print(F"Save dataset to {pytorch_dataset_dump_path}"	)
		torch.save(_UpperCamelCase ,						_UpperCamelCase	)

	if tf_checkpoint_path:
		# Convert a pre-trained TensorFlow model
		__lowerCAmelCase    :		Tuple		    =   os.path.abspath(_UpperCamelCase	)
		__lowerCAmelCase    :		List[Any]		    =   os.path.abspath(_UpperCamelCase	)

		print(F"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."	)
		# Initialise PyTorch model
		if transfo_xl_config_file == "":
			__lowerCAmelCase    :		Any		    =   TransfoXLConfig()
		else:
			__lowerCAmelCase    :		str		    =   TransfoXLConfig.from_json_file(_UpperCamelCase	)
		print(F"Building PyTorch model from configuration: {config}"	)
		__lowerCAmelCase    :		Optional[Any]		    =   TransfoXLLMHeadModel(_UpperCamelCase	)

		__lowerCAmelCase    :		Dict		    =   load_tf_weights_in_transfo_xl(_UpperCamelCase ,						_UpperCamelCase ,						_UpperCamelCase	)
		# Save pytorch-model
		__lowerCAmelCase    :		str		    =   os.path.join(_UpperCamelCase ,						_UpperCamelCase	)
		__lowerCAmelCase    :		Tuple		    =   os.path.join(_UpperCamelCase ,						_UpperCamelCase	)
		print(F"Save PyTorch model to {os.path.abspath(_UpperCamelCase	)}"	)
		torch.save(model.state_dict() ,						_UpperCamelCase	)
		print(F"Save configuration file to {os.path.abspath(_UpperCamelCase	)}"	)
		with open(_UpperCamelCase ,						'w' ,						encoding='utf-8'	) as f:
			f.write(config.to_json_string()	)


if __name__ == "__main__":
							lowerCamelCase__           =	argparse.ArgumentParser()
							parser.add_argument(
							    """--pytorch_dump_folder_path""",
							    default=None,
							    type=str,
							    required=True,
							    help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
							)
							parser.add_argument(
							    """--tf_checkpoint_path""",
							    default="""""",
							    type=str,
							    help="""An optional path to a TensorFlow checkpoint path to be converted.""",
							)
							parser.add_argument(
							    """--transfo_xl_config_file""",
							    default="""""",
							    type=str,
							    help=(
							        """An optional config json file corresponding to the pre-trained BERT model. \n"""
							        """This specifies the model architecture."""
							    ),
							)
							parser.add_argument(
							    """--transfo_xl_dataset_file""",
							    default="""""",
							    type=str,
							    help="""An optional dataset file to be converted in a vocabulary.""",
							)
							lowerCamelCase__           =	parser.parse_args()
							convert_transfo_xl_checkpoint_to_pytorch(
							    args.tf_checkpoint_path,
							    args.transfo_xl_config_file,
							    args.pytorch_dump_folder_path,
							    args.transfo_xl_dataset_file,
							)