

'''simple docstring'''





import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
		import torch

if is_vision_available():
		from PIL import Image

		from transformers import BeitImageProcessor





class     snake_case_			(				unittest.TestCase							):


							"""simple docstring"""





							def __init__( self					,						_A					,						_A=7					,						_A=3					,						_A=1_8					,						_A=3_0					,						_A=4_0_0					,						_A=True					,						_A=None					,						_A=True					,						_A=None					,						_A=True					,						_A=[0.5, 0.5, 0.5]					,						_A=[0.5, 0.5, 0.5]					,						_A=False					,						):
												__lowerCAmelCase					=							size if size is not None else {'''height''': 2_0, '''width''': 2_0}
												__lowerCAmelCase					=							crop_size if crop_size is not None else {'''height''': 1_8, '''width''': 1_8}
												__lowerCAmelCase					=							parent
												__lowerCAmelCase					=							batch_size
												__lowerCAmelCase					=							num_channels
												__lowerCAmelCase					=							image_size
												__lowerCAmelCase					=							min_resolution
												__lowerCAmelCase					=							max_resolution
												__lowerCAmelCase					=							do_resize
												__lowerCAmelCase					=							size
												__lowerCAmelCase					=							do_center_crop
												__lowerCAmelCase					=							crop_size
												__lowerCAmelCase					=							do_normalize
												__lowerCAmelCase					=							image_mean
												__lowerCAmelCase					=							image_std
												__lowerCAmelCase					=							do_reduce_labels




							def        A__  ( self							):
												return {
												    "do_resize": self.do_resize,
												    "size": self.size,
												    "do_center_crop": self.do_center_crop,
												    "crop_size": self.crop_size,
												    "do_normalize": self.do_normalize,
												    "image_mean": self.image_mean,
												    "image_std": self.image_std,
												    "do_reduce_labels": self.do_reduce_labels,
												}


def        __lowercase    (  ):

					"""simple docstring"""






					__lowerCAmelCase					=							load_dataset('hf-internal-testing/fixtures_ade20k'		,					split='test'      )

					__lowerCAmelCase					=							Image.open(dataset[0]['file']      )
					__lowerCAmelCase					=							Image.open(dataset[1]['file']      )

					return image, map


def        __lowercase    (  ):

					"""simple docstring"""






					__lowerCAmelCase					=							load_dataset('hf-internal-testing/fixtures_ade20k'		,					split='test'      )

					__lowerCAmelCase					=							Image.open(ds[0]['file']      )
					__lowerCAmelCase					=							Image.open(ds[1]['file']      )
					__lowerCAmelCase					=							Image.open(ds[2]['file']      )
					__lowerCAmelCase					=							Image.open(ds[3]['file']      )

					return [imagea, imagea], [mapa, mapa]





@require_torch
@require_vision
class     snake_case_			(				SCREAMING_SNAKE_CASE_			, unittest.TestCase							):


							"""simple docstring"""





							__UpperCAmelCase       =BeitImageProcessor if is_vision_available() else None




							def        A__  ( self							):
												__lowerCAmelCase					=							BeitImageProcessingTester(self							)




							@property
							def        A__  ( self							):
												return self.image_processor_tester.prepare_image_processor_dict()




							def        A__  ( self							):
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'do_resize'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'size'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'do_center_crop'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'center_crop'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'do_normalize'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'image_mean'							)							)
												self.assertTrue(hasattr(UpperCamelCase__					,						'image_std'							)							)




							def        A__  ( self							):
												__lowerCAmelCase					=							self.image_processing_class.from_dict(self.image_processor_dict							)
												self.assertEqual(image_processor.size					,						{'height': 2_0, 'width': 2_0}							)
												self.assertEqual(image_processor.crop_size					,						{'height': 1_8, 'width': 1_8}							)
												self.assertEqual(image_processor.do_reduce_labels					,						UpperCamelCase__							)

												__lowerCAmelCase					=							self.image_processing_class.from_dict(
												    self.image_processor_dict					,						size=4_2					,						crop_size=8_4					,						reduce_labels=UpperCamelCase__							)
												self.assertEqual(image_processor.size					,						{'height': 4_2, 'width': 4_2}							)
												self.assertEqual(image_processor.crop_size					,						{'height': 8_4, 'width': 8_4}							)
												self.assertEqual(image_processor.do_reduce_labels					,						UpperCamelCase__							)




							def        A__  ( self							):
												pass




							def        A__  ( self							):
												# Initialize image_processing
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)
												# create random PIL images
												__lowerCAmelCase					=							prepare_image_inputs(self.image_processor_tester					,						equal_resolution=UpperCamelCase__							)
												for image in image_inputs:
																	self.assertIsInstance(UpperCamelCase__					,						Image.Image							)

												# Test not batched input
												__lowerCAmelCase					=							image_processing(image_inputs[0]					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        1,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)

												# Test batched
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        self.image_processor_tester.batch_size,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)




							def        A__  ( self							):
												# Initialize image_processing
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)
												# create random numpy tensors
												__lowerCAmelCase					=							prepare_image_inputs(self.image_processor_tester					,						equal_resolution=UpperCamelCase__					,						numpify=UpperCamelCase__							)
												for image in image_inputs:
																	self.assertIsInstance(UpperCamelCase__					,						np.ndarray							)

												# Test not batched input
												__lowerCAmelCase					=							image_processing(image_inputs[0]					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        1,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)

												# Test batched
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        self.image_processor_tester.batch_size,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)




							def        A__  ( self							):
												# Initialize image_processing
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)
												# create random PyTorch tensors
												__lowerCAmelCase					=							prepare_image_inputs(self.image_processor_tester					,						equal_resolution=UpperCamelCase__					,						torchify=UpperCamelCase__							)
												for image in image_inputs:
																	self.assertIsInstance(UpperCamelCase__					,						torch.Tensor							)

												# Test not batched input
												__lowerCAmelCase					=							image_processing(image_inputs[0]					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        1,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)

												# Test batched
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						return_tensors='pt'							).pixel_values
												self.assertEqual(
												    encoded_images.shape					,						(
												        self.image_processor_tester.batch_size,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)




							def        A__  ( self							):
												# Initialize image_processing
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)
												# create random PyTorch tensors
												__lowerCAmelCase					=							prepare_image_inputs(self.image_processor_tester					,						equal_resolution=UpperCamelCase__					,						torchify=UpperCamelCase__							)
												__lowerCAmelCase					=							[]
												for image in image_inputs:
																	self.assertIsInstance(UpperCamelCase__					,						torch.Tensor							)
																	maps.append(torch.zeros(image.shape[-2:]							).long()							)

												# Test not batched input
												__lowerCAmelCase					=							image_processing(image_inputs[0]					,						maps[0]					,						return_tensors='pt'							)
												self.assertEqual(
												    encoding['pixel_values'].shape					,						(
												        1,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(
												    encoding['labels'].shape					,						(
												        1,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(encoding['labels'].dtype					,						torch.long							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 2_5_5							)

												# Test batched
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						UpperCamelCase__					,						return_tensors='pt'							)
												self.assertEqual(
												    encoding['pixel_values'].shape					,						(
												        self.image_processor_tester.batch_size,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(
												    encoding['labels'].shape					,						(
												        self.image_processor_tester.batch_size,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(encoding['labels'].dtype					,						torch.long							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 2_5_5							)

												# Test not batched input (PIL images)
												__lowerCAmelCase					=							prepare_semantic_single_inputs()

												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						UpperCamelCase__					,						return_tensors='pt'							)
												self.assertEqual(
												    encoding['pixel_values'].shape					,						(
												        1,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(
												    encoding['labels'].shape					,						(
												        1,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(encoding['labels'].dtype					,						torch.long							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 2_5_5							)

												# Test batched input (PIL images)
												__lowerCAmelCase					=							prepare_semantic_batch_inputs()

												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						UpperCamelCase__					,						return_tensors='pt'							)
												self.assertEqual(
												    encoding['pixel_values'].shape					,						(
												        2,
												        self.image_processor_tester.num_channels,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(
												    encoding['labels'].shape					,						(
												        2,
												        self.image_processor_tester.crop_size['height'],
												        self.image_processor_tester.crop_size['width'],
												    )					,						)
												self.assertEqual(encoding['labels'].dtype					,						torch.long							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 2_5_5							)




							def        A__  ( self							):
												# Initialize image_processing
												__lowerCAmelCase					=							self.image_processing_class(**self.image_processor_dict							)

												# ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
												__lowerCAmelCase					=							prepare_semantic_single_inputs()
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						UpperCamelCase__					,						return_tensors='pt'							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 1_5_0							)

												__lowerCAmelCase					=							True
												__lowerCAmelCase					=							image_processing(UpperCamelCase__					,						UpperCamelCase__					,						return_tensors='pt'							)
												self.assertTrue(encoding['labels'].min().item() >= 0							)
												self.assertTrue(encoding['labels'].max().item() <= 2_5_5							)









import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowerCamelCase      				=     data_utils.TransfoXLTokenizer
lowerCamelCase      				=     data_utils.TransfoXLCorpus
lowerCamelCase      				=     data_utils
lowerCamelCase      				=     data_utils


def        __lowercase    (  UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__      ):

					"""simple docstring"""






					if transfo_xl_dataset_file:
										# Convert a pre-processed corpus (see original TensorFlow repo)
										with open(UpperCAmelCase__		,					'rb'      ) as fp:
															__lowerCAmelCase					=							pickle.load(UpperCAmelCase__		,					encoding='latin1'      )
										# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
										__lowerCAmelCase					=							pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
										print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""      )
										__lowerCAmelCase					=							corpus.vocab.__dict__
										torch.save(UpperCAmelCase__		,					UpperCAmelCase__      )

										__lowerCAmelCase					=							corpus.__dict__
										corpus_dict_no_vocab.pop('vocab'		,					UpperCAmelCase__      )
										__lowerCAmelCase					=							pytorch_dump_folder_path + '/' + CORPUS_NAME
										print(F"""Save dataset to {pytorch_dataset_dump_path}"""      )
										torch.save(UpperCAmelCase__		,					UpperCAmelCase__      )

					if tf_checkpoint_path:
										# Convert a pre-trained TensorFlow model
										__lowerCAmelCase					=							os.path.abspath(UpperCAmelCase__      )
										__lowerCAmelCase					=							os.path.abspath(UpperCAmelCase__      )

										print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""      )
										# Initialise PyTorch model
										if transfo_xl_config_file == "":
															__lowerCAmelCase					=							TransfoXLConfig()
										else:
															__lowerCAmelCase					=							TransfoXLConfig.from_json_file(UpperCAmelCase__      )
										print(F"""Building PyTorch model from configuration: {config}"""      )
										__lowerCAmelCase					=							TransfoXLLMHeadModel(UpperCAmelCase__      )

										__lowerCAmelCase					=							load_tf_weights_in_transfo_xl(UpperCAmelCase__		,					UpperCAmelCase__		,					UpperCAmelCase__      )
										# Save pytorch-model
										__lowerCAmelCase					=							os.path.join(UpperCAmelCase__		,					UpperCAmelCase__      )
										__lowerCAmelCase					=							os.path.join(UpperCAmelCase__		,					UpperCAmelCase__      )
										print(F"""Save PyTorch model to {os.path.abspath(UpperCAmelCase__      )}"""      )
										torch.save(model.state_dict()		,					UpperCAmelCase__      )
										print(F"""Save configuration file to {os.path.abspath(UpperCAmelCase__      )}"""      )
										with open(UpperCAmelCase__		,					'w'		,					encoding='utf-8'      ) as f:
															f.write(config.to_json_string()      )


if __name__ == "__main__":
		lowerCamelCase      				=     argparse.ArgumentParser()
		parser.add_argument(
		    '''--pytorch_dump_folder_path''',
		    default=None,
		    type=str,
		    required=True,
		    help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
		)
		parser.add_argument(
		    '''--tf_checkpoint_path''',
		    default='''''',
		    type=str,
		    help='''An optional path to a TensorFlow checkpoint path to be converted.''',
		)
		parser.add_argument(
		    '''--transfo_xl_config_file''',
		    default='''''',
		    type=str,
		    help=(
		        '''An optional config json file corresponding to the pre-trained BERT model. \n'''
		        '''This specifies the model architecture.'''
		    ),
		)
		parser.add_argument(
		    '''--transfo_xl_dataset_file''',
		    default='''''',
		    type=str,
		    help='''An optional dataset file to be converted in a vocabulary.''',
		)
		lowerCamelCase      				=     parser.parse_args()
		convert_transfo_xl_checkpoint_to_pytorch(
		    args.tf_checkpoint_path,
		    args.transfo_xl_config_file,
		    args.pytorch_dump_folder_path,
		    args.transfo_xl_dataset_file,
		)






