





"""simple docstring"""







import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoImageProcessor, ViTImageProcessor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


a      =							get_tests_dir('fixtures')






class       SCREAMING_SNAKE_CASE__					(					unittest.TestCase					):


						def   __lowercase (							self				:			Optional[Any]					):
										# A mock response for an HTTP head request to emulate server down
										lowerCAmelCase  =				mock.Mock()
										lowerCAmelCase  =				500
										lowerCAmelCase  =				{}
										lowerCAmelCase  =				HTTPError
										lowerCAmelCase  =				{}

										# Download this model to make sure it's in the cache.
										lowerCAmelCase  =				ViTImageProcessor.from_pretrained("""hf-internal-testing/tiny-random-vit"""					)
										# Under the mock environment we get a 500 error when trying to reach the model.
										with mock.patch("""requests.Session.request"""						,						return_value=_A					) as mock_head:
														lowerCAmelCase  =				ViTImageProcessor.from_pretrained("""hf-internal-testing/tiny-random-vit"""					)
														# This check we did call the fake head request
														mock_head.assert_called()


						def   __lowercase (							self				:			Tuple					):
										# This test is for deprecated behavior and can be removed in v5
										lowerCAmelCase  =				ViTImageProcessor.from_pretrained(
										    """https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json"""					)




						def   __lowercase (							self				:			str					):
										with self.assertRaises(_A					):
														# config is in subfolder, the following should not work without specifying the subfolder
														lowerCAmelCase  =				AutoImageProcessor.from_pretrained("""hf-internal-testing/stable-diffusion-all-variants"""					)

										lowerCAmelCase  =				AutoImageProcessor.from_pretrained(
										    """hf-internal-testing/stable-diffusion-all-variants"""						,						subfolder="""feature_extractor"""					)

										self.assertIsNotNone(_A					)






@is_staging_test
class       SCREAMING_SNAKE_CASE__					(					unittest.TestCase					):


						@classmethod
						def   __lowercase (							cls				:			Optional[int]					):
										lowerCAmelCase  =				TOKEN
										HfFolder.save_token(_A					)


						@classmethod
						def   __lowercase (							cls				:			Optional[int]					):
										try:
														delete_repo(token=cls._token						,						repo_id="""test-image-processor"""					)
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token						,						repo_id="""valid_org/test-image-processor-org"""					)
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token						,						repo_id="""test-dynamic-image-processor"""					)
										except HTTPError:
														pass


						def   __lowercase (							self				:			int					):
										lowerCAmelCase  =				ViTImageProcessor.from_pretrained(_A					)
										image_processor.push_to_hub("""test-image-processor"""						,						use_auth_token=self._token					)

										lowerCAmelCase  =				ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''					)
										for k, v in image_processor.__dict__.items():
														self.assertEqual(_A						,						getattr(_A						,						_A					)					)

										# Reset repo
										delete_repo(token=self._token						,						repo_id="""test-image-processor"""					)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														image_processor.save_pretrained(
														    _A						,						repo_id="""test-image-processor"""						,						push_to_hub=_A						,						use_auth_token=self._token					)

										lowerCAmelCase  =				ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''					)
										for k, v in image_processor.__dict__.items():
														self.assertEqual(_A						,						getattr(_A						,						_A					)					)


						def   __lowercase (							self				:			Dict					):
										lowerCAmelCase  =				ViTImageProcessor.from_pretrained(_A					)
										image_processor.push_to_hub("""valid_org/test-image-processor"""						,						use_auth_token=self._token					)

										lowerCAmelCase  =				ViTImageProcessor.from_pretrained("""valid_org/test-image-processor"""					)
										for k, v in image_processor.__dict__.items():
														self.assertEqual(_A						,						getattr(_A						,						_A					)					)

										# Reset repo
										delete_repo(token=self._token						,						repo_id="""valid_org/test-image-processor"""					)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														image_processor.save_pretrained(
														    _A						,						repo_id="""valid_org/test-image-processor-org"""						,						push_to_hub=_A						,						use_auth_token=self._token					)

										lowerCAmelCase  =				ViTImageProcessor.from_pretrained("""valid_org/test-image-processor-org"""					)
										for k, v in image_processor.__dict__.items():
														self.assertEqual(_A						,						getattr(_A						,						_A					)					)




						def   __lowercase (							self				:			List[Any]					):
										CustomImageProcessor.register_for_auto_class()
										lowerCAmelCase  =				CustomImageProcessor.from_pretrained(_A					)

										image_processor.push_to_hub("""test-dynamic-image-processor"""						,						use_auth_token=self._token					)

										# This has added the proper auto_map field to the config
										self.assertDictEqual(
										    image_processor.auto_map						,						{"""AutoImageProcessor""": """custom_image_processing.CustomImageProcessor"""}						,						)

										lowerCAmelCase  =				AutoImageProcessor.from_pretrained(
										    f'''{USER}/test-dynamic-image-processor'''						,						trust_remote_code=_A					)
										# Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module
										self.assertEqual(new_image_processor.__class__.__name__						,						"""CustomImageProcessor"""					)











"""simple docstring"""







import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()





def 							lowercase					(snake_case__   : int      ,       snake_case__   : Union[str, Any] )     ->  List[Any]:


				'''simple docstring'''






				lowerCAmelCase  =				torch.load(snake_case__      ,       map_location="""cpu""" )

				lowerCAmelCase  =				chkpt["""model"""]

				# We have the base model one level deeper than the original XLM repository
				lowerCAmelCase  =				{}
				for k, v in state_dict.items():
								if "pred_layer" in k:
												lowerCAmelCase  =				v
								else:
												lowerCAmelCase  =				v

				lowerCAmelCase  =				chkpt["""params"""]
				lowerCAmelCase  =				{n: v for n, v in config.items() if not isinstance(snake_case__      ,       (torch.FloatTensor, numpy.ndarray) )}

				lowerCAmelCase  =				chkpt["""dico_word2id"""]
				lowerCAmelCase  =				{s + """</w>""" if s.find("""@@""" ) == -1 and i > 13 else s.replace("""@@"""      ,       """""" ): i for s, i in vocab.items()}

				# Save pytorch-model
				lowerCAmelCase  =				pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
				lowerCAmelCase  =				pytorch_dump_folder_path + """/""" + CONFIG_NAME
				lowerCAmelCase  =				pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""vocab_file"""]

				print(f'''Save PyTorch model to {pytorch_weights_dump_path}''' )
				torch.save(snake_case__      ,       snake_case__ )

				print(f'''Save configuration file to {pytorch_config_dump_path}''' )
				with open(snake_case__      ,       """w"""      ,       encoding="""utf-8""" ) as f:
								f.write(json.dumps(snake_case__      ,       indent=2 ) + """\n""" )

				print(f'''Save vocab file to {pytorch_config_dump_path}''' )
				with open(snake_case__      ,       """w"""      ,       encoding="""utf-8""" ) as f:
								f.write(json.dumps(snake_case__      ,       indent=2 ) + """\n""" )


if __name__ == "__main__":
		a      =							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--xlm_checkpoint_path', default=None, type=str, required=True, help='Path the official PyTorch dump.'
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		a      =							parser.parse_args()
		convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)





