

'''simple docstring'''





import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoImageProcessor, ViTImageProcessor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


lowerCamelCase      				=     get_tests_dir('''fixtures''')





class     snake_case_			(				unittest.TestCase							):


							"""simple docstring"""





							def        A__  ( self							):
												# A mock response for an HTTP head request to emulate server down
												__lowerCAmelCase					=							mock.Mock()
												__lowerCAmelCase					=							5_0_0
												__lowerCAmelCase					=							{}
												__lowerCAmelCase					=							HTTPError
												__lowerCAmelCase					=							{}

												# Download this model to make sure it's in the cache.
												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained('hf-internal-testing/tiny-random-vit'							)
												# Under the mock environment we get a 500 error when trying to reach the model.
												with mock.patch('requests.Session.request'					,						return_value=_A							) as mock_head:
																	__lowerCAmelCase					=							ViTImageProcessor.from_pretrained('hf-internal-testing/tiny-random-vit'							)
																	# This check we did call the fake head request
																	mock_head.assert_called()




							def        A__  ( self							):
												# This test is for deprecated behavior and can be removed in v5
												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained(
												    'https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json'							)




							def        A__  ( self							):
												with self.assertRaises(_A							):
																	# config is in subfolder, the following should not work without specifying the subfolder
																	__lowerCAmelCase					=							AutoImageProcessor.from_pretrained('hf-internal-testing/stable-diffusion-all-variants'							)

												__lowerCAmelCase					=							AutoImageProcessor.from_pretrained(
												    'hf-internal-testing/stable-diffusion-all-variants'					,						subfolder='feature_extractor'							)

												self.assertIsNotNone(_A							)





@is_staging_test
class     snake_case_			(				unittest.TestCase							):


							"""simple docstring"""





							@classmethod
							def        A__  ( cls							):
												__lowerCAmelCase					=							TOKEN
												HfFolder.save_token(_A							)




							@classmethod
							def        A__  ( cls							):
												try:
																	delete_repo(token=cls._token					,						repo_id='test-image-processor'							)
												except HTTPError:
																	pass

												try:
																	delete_repo(token=cls._token					,						repo_id='valid_org/test-image-processor-org'							)
												except HTTPError:
																	pass

												try:
																	delete_repo(token=cls._token					,						repo_id='test-dynamic-image-processor'							)
												except HTTPError:
																	pass




							def        A__  ( self							):
												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained(_A							)
												image_processor.push_to_hub('test-image-processor'					,						use_auth_token=self._token							)

												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained(F"""{USER}/test-image-processor"""							)
												for k, v in image_processor.__dict__.items():
																	self.assertEqual(_A					,						getattr(_A					,						_A							)							)

												# Reset repo
												delete_repo(token=self._token					,						repo_id='test-image-processor'							)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																	image_processor.save_pretrained(
																	    _A					,						repo_id='test-image-processor'					,						push_to_hub=_A					,						use_auth_token=self._token							)

												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained(F"""{USER}/test-image-processor"""							)
												for k, v in image_processor.__dict__.items():
																	self.assertEqual(_A					,						getattr(_A					,						_A							)							)




							def        A__  ( self							):
												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained(_A							)
												image_processor.push_to_hub('valid_org/test-image-processor'					,						use_auth_token=self._token							)

												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained('valid_org/test-image-processor'							)
												for k, v in image_processor.__dict__.items():
																	self.assertEqual(_A					,						getattr(_A					,						_A							)							)

												# Reset repo
												delete_repo(token=self._token					,						repo_id='valid_org/test-image-processor'							)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																	image_processor.save_pretrained(
																	    _A					,						repo_id='valid_org/test-image-processor-org'					,						push_to_hub=_A					,						use_auth_token=self._token							)

												__lowerCAmelCase					=							ViTImageProcessor.from_pretrained('valid_org/test-image-processor-org'							)
												for k, v in image_processor.__dict__.items():
																	self.assertEqual(_A					,						getattr(_A					,						_A							)							)




							def        A__  ( self							):
												CustomImageProcessor.register_for_auto_class()
												__lowerCAmelCase					=							CustomImageProcessor.from_pretrained(_A							)

												image_processor.push_to_hub('test-dynamic-image-processor'					,						use_auth_token=self._token							)

												# This has added the proper auto_map field to the config
												self.assertDictEqual(
												    image_processor.auto_map					,						{'AutoImageProcessor': 'custom_image_processing.CustomImageProcessor'}					,						)

												__lowerCAmelCase					=							AutoImageProcessor.from_pretrained(
												    F"""{USER}/test-dynamic-image-processor"""					,						trust_remote_code=_A							)
												# Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module
												self.assertEqual(new_image_processor.__class__.__name__					,						'CustomImageProcessor'							)









import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCamelCase      				=     data_utils.TransfoXLTokenizer
lowerCamelCase      				=     data_utils.TransfoXLCorpus
lowerCamelCase      				=     data_utils
lowerCamelCase      				=     data_utils


def        __lowercase    (  UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__      ):

					"""simple docstring"""






					if transfo_xl_dataset_file:
										# Convert a pre-processed corpus (see original TensorFlow repo)
										with open(UpperCAmelCase__		,					'rb'      ) as fp:
															__lowerCAmelCase					=							pickle.load(UpperCAmelCase__		,					encoding='latin1'      )
										# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
										__lowerCAmelCase					=							pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
										print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""      )
										__lowerCAmelCase					=							corpus.vocab.__dict__
										torch.save(UpperCAmelCase__		,					UpperCAmelCase__      )

										__lowerCAmelCase					=							corpus.__dict__
										corpus_dict_no_vocab.pop('vocab'		,					UpperCAmelCase__      )
										__lowerCAmelCase					=							pytorch_dump_folder_path + '/' + CORPUS_NAME
										print(F"""Save dataset to {pytorch_dataset_dump_path}"""      )
										torch.save(UpperCAmelCase__		,					UpperCAmelCase__      )

					if tf_checkpoint_path:
										# Convert a pre-trained TensorFlow model
										__lowerCAmelCase					=							os.path.abspath(UpperCAmelCase__      )
										__lowerCAmelCase					=							os.path.abspath(UpperCAmelCase__      )

										print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""      )
										# Initialise PyTorch model
										if transfo_xl_config_file == "":
															__lowerCAmelCase					=							TransfoXLConfig()
										else:
															__lowerCAmelCase					=							TransfoXLConfig.from_json_file(UpperCAmelCase__      )
										print(F"""Building PyTorch model from configuration: {config}"""      )
										__lowerCAmelCase					=							TransfoXLLMHeadModel(UpperCAmelCase__      )

										__lowerCAmelCase					=							load_tf_weights_in_transfo_xl(UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__      )
										# Save pytorch-model
										__lowerCAmelCase					=							os.path.join(UpperCAmelCase__		,					UpperCAmelCase__      )
										__lowerCAmelCase					=							os.path.join(UpperCAmelCase__		,					UpperCAmelCase__      )
										print(F"""Save PyTorch model to {os.path.abspath(UpperCAmelCase__      )}"""      )
										torch.save(model.state_dict()		,					UpperCAmelCase__      )
										print(F"""Save configuration file to {os.path.abspath(UpperCAmelCase__      )}"""      )
										with open(UpperCAmelCase__		,					'w'		,					encoding='utf-8'      ) as f:
															f.write(config.to_json_string()      )


if __name__ == "__main__":
		lowerCamelCase      				=     argparse.ArgumentParser()
		parser.add_argument(
		    '''--pytorch_dump_folder_path''',
		    default=None,
		    type=str,
		    required=True,
		    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
		)
		parser.add_argument(
		    '''--tf_checkpoint_path''',
		    default='''''',
		    type=str,
		    help='''An optional path to a TensorFlow checkpoint path to be converted.''',
		)
		parser.add_argument(
		    '''--transfo_xl_config_file''',
		    default='''''',
		    type=str,
		    help=(
		        '''An optional config json file corresponding to the pre-trained BERT model. \n'''
		        '''This specifies the model architecture.'''
		    ),
		)
		parser.add_argument(
		    '''--transfo_xl_dataset_file''',
		    default='''''',
		    type=str,
		    help='''An optional dataset file to be converted in a vocabulary.''',
		)
		lowerCamelCase      				=     parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)






