




import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict







@pytest.mark.parametrize(
    'files'					,	[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]					,	)
def      lowerCamelCase_						(			UpperCamelCase__							: Tuple					,	UpperCamelCase__							: str )	->			int:





				"""simple docstring"""


				__lowerCamelCase									=       tmp_path_factory.mktemp('dset_infos_dir' )
				if "full:README.md" in files:
								with open(dataset_infos_dir / 'README.md'					,	'w' ) as f:
												f.write('---\ndataset_info:\n  dataset_size: 42\n---' )
				if "empty:README.md" in files:
								with open(dataset_infos_dir / 'README.md'					,	'w' ) as f:
												f.write('' )
    # we want to support dataset_infos.json for backward compatibility
				if "dataset_infos.json" in files:
								with open(dataset_infos_dir / 'dataset_infos.json'					,	'w' ) as f:
												f.write('{"default": {"dataset_size": 42}}' )
				__lowerCamelCase									=       DatasetInfosDict.from_directory(UpperCamelCase__ )
				assert dataset_infos
				assert dataset_infos["default"].dataset_size == 42







@pytest.mark.parametrize(
    'dataset_info'					,	[
        DatasetInfo(),
        DatasetInfo(
            description='foo'					,	features=Features({'a': Value('int32' )} )					,	builder_name='builder'					,	config_name='config'					,	version='1.0.0'					,	splits=[{'name': 'train'}]					,	download_size=42					,	),
    ]					,	)
def      lowerCamelCase_						(			UpperCamelCase__							: Optional[int]					,	UpperCamelCase__							: DatasetInfo )	->			Any:





				"""simple docstring"""


				__lowerCamelCase									=       str(UpperCamelCase__ )
				dataset_info.write_to_directory(UpperCamelCase__ )
				__lowerCamelCase									=       DatasetInfo.from_directory(UpperCamelCase__ )
				assert dataset_info == reloaded
				assert os.path.exists(os.path.join(UpperCamelCase__					,	'dataset_info.json' ) )







def      lowerCamelCase_						(			)	->			List[str]:





				"""simple docstring"""


				__lowerCamelCase									=       DatasetInfo(
				    description='foo'					,	citation='bar'					,	homepage='https://foo.bar'					,	license='CC0'					,	features=Features({'a': Value('int32' )} )					,	post_processed={}					,	supervised_keys=()					,	task_templates=[]					,	builder_name='builder'					,	config_name='config'					,	version='1.0.0'					,	splits=[{'name': 'train', 'num_examples': 42}]					,	download_checksums={}					,	download_size=1337					,	post_processing_size=442					,	dataset_size=1234					,	size_in_bytes=1337 + 442 + 1234					,	)
				__lowerCamelCase									=       dataset_info._to_yaml_dict()
				assert sorted(UpperCamelCase__ ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML )
				for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
								assert key in dataset_info_yaml_dict
								assert isinstance(dataset_info_yaml_dict[key]					,	(list, dict, int, str) )
				__lowerCamelCase									=       yaml.safe_dump(UpperCamelCase__ )
				__lowerCamelCase									=       yaml.safe_load(UpperCamelCase__ )
				assert dataset_info_yaml_dict == reloaded







def      lowerCamelCase_						(			)	->			Tuple:





				"""simple docstring"""


				__lowerCamelCase									=       DatasetInfo()
				__lowerCamelCase									=       dataset_info._to_yaml_dict()
				assert dataset_info_yaml_dict == {}







@pytest.mark.parametrize(
    'dataset_infos_dict'					,	[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()} ),
        DatasetInfosDict({'my_config_name': DatasetInfo()} ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'					,	features=Features({'a': Value('int32' )} )					,	builder_name='builder'					,	config_name='config'					,	version='1.0.0'					,	splits=[{'name': 'train'}]					,	download_size=42					,	)
            } ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42 ),
                'v2': DatasetInfo(dataset_size=1337 ),
            } ),
    ]					,	)
def      lowerCamelCase_						(			UpperCamelCase__							: Dict					,	UpperCamelCase__							: DatasetInfosDict )	->			List[str]:





				"""simple docstring"""


				__lowerCamelCase									=       str(UpperCamelCase__ )
				dataset_infos_dict.write_to_directory(UpperCamelCase__ )
				__lowerCamelCase									=       DatasetInfosDict.from_directory(UpperCamelCase__ )

				# the config_name of the dataset_infos_dict take over the attribute
				for config_name, dataset_info in dataset_infos_dict.items():
								__lowerCamelCase									=       config_name
								# the yaml representation doesn't include fields like description or citation
								# so we just test that we can recover what we can from the yaml
								__lowerCamelCase									=       DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict() )
				assert dataset_infos_dict == reloaded

				if dataset_infos_dict:
								assert os.path.exists(os.path.join(UpperCamelCase__					,	'README.md' ) )








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__A								=       data_utils.TransfoXLTokenizer
__A								=       data_utils.TransfoXLCorpus
__A								=       data_utils
__A								=       data_utils







def      lowerCamelCase_						(			UpperCamelCase__							: int					,	UpperCamelCase__							: str					,	UpperCamelCase__							: Optional[Any]					,	UpperCamelCase__							: Tuple )	->			Tuple:





				"""simple docstring"""


				if transfo_xl_dataset_file:
								# Convert a pre-processed corpus (see original TensorFlow repo)
								with open(UpperCamelCase__					,	'rb' ) as fp:
												__lowerCamelCase									=       pickle.load(UpperCamelCase__					,	encoding='latin1' )
								# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
								__lowerCamelCase									=       pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
								print(F"""Save vocabulary to {pytorch_vocab_dump_path}""" )
								__lowerCamelCase									=       corpus.vocab.__dict__
								torch.save(UpperCamelCase__					,	UpperCamelCase__ )

								__lowerCamelCase									=       corpus.__dict__
								corpus_dict_no_vocab.pop('vocab'					,	UpperCamelCase__ )
								__lowerCamelCase									=       pytorch_dump_folder_path + '/' + CORPUS_NAME
								print(F"""Save dataset to {pytorch_dataset_dump_path}""" )
								torch.save(UpperCamelCase__					,	UpperCamelCase__ )

				if tf_checkpoint_path:
								# Convert a pre-trained TensorFlow model
								__lowerCamelCase									=       os.path.abspath(UpperCamelCase__ )
								__lowerCamelCase									=       os.path.abspath(UpperCamelCase__ )

								print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.""" )
								# Initialise PyTorch model
								if transfo_xl_config_file == "":
												__lowerCamelCase									=       TransfoXLConfig()
								else:
												__lowerCamelCase									=       TransfoXLConfig.from_json_file(UpperCamelCase__ )
								print(F"""Building PyTorch model from configuration: {config}""" )
								__lowerCamelCase									=       TransfoXLLMHeadModel(UpperCamelCase__ )

								__lowerCamelCase									=       load_tf_weights_in_transfo_xl(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )
								# Save pytorch-model
								__lowerCamelCase									=       os.path.join(UpperCamelCase__					,	UpperCamelCase__ )
								__lowerCamelCase									=       os.path.join(UpperCamelCase__					,	UpperCamelCase__ )
								print(F"""Save PyTorch model to {os.path.abspath(UpperCamelCase__ )}""" )
								torch.save(model.state_dict()					,	UpperCamelCase__ )
								print(F"""Save configuration file to {os.path.abspath(UpperCamelCase__ )}""" )
								with open(UpperCamelCase__					,	'w'					,	encoding='utf-8' ) as f:
												f.write(config.to_json_string() )


if __name__ == "__main__":
					__A								=       argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					__A								=       parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)


