


'''simple docstring'''







import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
       import torch

       from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
       lowerCAmelCase     :  str											=    False

if is_vision_available():
       from PIL import Image

       from transformers import PixaStructImageProcessor
class 						UpperCamelCase__       (			unittest.TestCase						):



     """simple docstring"""






     def __init__(			self   ,						snake_case__   ,						snake_case__=7   ,						snake_case__=3   ,						snake_case__=18   ,						snake_case__=30   ,						snake_case__=400   ,						snake_case__=None   ,						snake_case__=True   ,						snake_case__=True   ,						snake_case__=None   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							size if size is not None else {"height": 20, "width": 20}
            _lowerCAmelCase     :				Optional[int]       =							parent
            _lowerCAmelCase     :				Union[str, Any]       =							batch_size
            _lowerCAmelCase     :				List[Any]       =							num_channels
            _lowerCAmelCase     :				Union[str, Any]       =							image_size
            _lowerCAmelCase     :				List[str]       =							min_resolution
            _lowerCAmelCase     :				int       =							max_resolution
            _lowerCAmelCase     :				int       =							size
            _lowerCAmelCase     :				List[str]       =							do_normalize
            _lowerCAmelCase     :				Any       =							do_convert_rgb
            _lowerCAmelCase     :				Optional[Any]       =							[512, 1024, 2048, 4096]
            _lowerCAmelCase     :				Any       =							patch_size if patch_size is not None else {"height": 16, "width": 16}







     def a						(			self ):

            '''simple docstring'''

            return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg"
            _lowerCAmelCase     :				Union[str, Any]       =							Image.open(requests.get(__UpperCAmelCase   ,						stream=__UpperCAmelCase ).raw ).convert('RGB' )
            return raw_image
@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11     ,       reason="`Pix2StructImageProcessor` requires `torch>=1.11.0`."     ,       )
@require_torch
@require_vision
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_     ,       unittest.TestCase						):



     """simple docstring"""






     __magic_name__           =     PixaStructImageProcessor if is_vision_available() else None







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							PixaStructImageProcessingTester(self )







     @property
     def a						(			self ):

            '''simple docstring'''

            return self.image_processor_tester.prepare_image_processor_dict()







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Union[str, Any]       =							self.image_processing_class(**self.image_processor_dict )
            self.assertTrue(hasattr(__UpperCAmelCase   ,						'do_normalize' ) )
            self.assertTrue(hasattr(__UpperCAmelCase   ,						'do_convert_rgb' ) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							self.image_processor_tester.prepare_dummy_image()

            _lowerCAmelCase     :				Any       =							self.image_processing_class(**self.image_processor_dict )
            _lowerCAmelCase     :				List[str]       =							2048

            _lowerCAmelCase     :				Union[str, Any]       =							image_processor(__UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase )
            self.assertTrue(torch.allclose(inputs.flattened_patches.mean()   ,						torch.tensor(0.0606 )   ,						atol=1E-3   ,						rtol=1E-3 ) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							self.image_processing_class(**self.image_processor_dict )
            # create random PIL images
            _lowerCAmelCase     :				str       =							prepare_image_inputs(self.image_processor_tester   ,						equal_resolution=__UpperCAmelCase )
            for image in image_inputs:
                   self.assertIsInstance(__UpperCAmelCase   ,						Image.Image )

            # Test not batched input
            _lowerCAmelCase     :				List[Any]       =							(
                (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"])
                * self.image_processor_tester.num_channels
            ) + 2

            for max_patch in self.image_processor_tester.max_patches:
                   # Test not batched input
                   _lowerCAmelCase     :				Optional[Any]       =							image_processor(
                       image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(1, max_patch, expected_hidden_dim)   ,						)

                   # Test batched
                   _lowerCAmelCase     :				Optional[Any]       =							image_processor(
                       __UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)   ,						)







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Union[str, Any]       =							self.image_processing_class(**self.image_processor_dict )
            # create random PIL images
            _lowerCAmelCase     :				Any       =							prepare_image_inputs(self.image_processor_tester   ,						equal_resolution=__UpperCAmelCase )
            for image in image_inputs:
                   self.assertIsInstance(__UpperCAmelCase   ,						Image.Image )

            # Test not batched input
            _lowerCAmelCase     :				Optional[int]       =							(
                (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"])
                * self.image_processor_tester.num_channels
            ) + 2

            _lowerCAmelCase     :				int       =							True

            for max_patch in self.image_processor_tester.max_patches:
                   # Test not batched input
                   with self.assertRaises(__UpperCAmelCase ):
                          _lowerCAmelCase     :				List[str]       =							image_processor(
                              image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches

                   _lowerCAmelCase     :				Tuple       =							"Hello"

                   _lowerCAmelCase     :				Union[str, Any]       =							image_processor(
                       image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase   ,						header_text=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(1, max_patch, expected_hidden_dim)   ,						)

                   # Test batched
                   _lowerCAmelCase     :				Optional[int]       =							image_processor(
                       __UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase   ,						header_text=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)   ,						)







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							self.image_processing_class(**self.image_processor_dict )
            # create random numpy tensors
            _lowerCAmelCase     :				Tuple       =							prepare_image_inputs(self.image_processor_tester   ,						equal_resolution=__UpperCAmelCase   ,						numpify=__UpperCAmelCase )
            for image in image_inputs:
                   self.assertIsInstance(__UpperCAmelCase   ,						np.ndarray )

            _lowerCAmelCase     :				List[str]       =							(
                (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"])
                * self.image_processor_tester.num_channels
            ) + 2

            for max_patch in self.image_processor_tester.max_patches:
                   # Test not batched input
                   _lowerCAmelCase     :				Any       =							image_processor(
                       image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(1, max_patch, expected_hidden_dim)   ,						)

                   # Test batched
                   _lowerCAmelCase     :				str       =							image_processor(
                       __UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)   ,						)







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							self.image_processing_class(**self.image_processor_dict )
            # create random PyTorch tensors
            _lowerCAmelCase     :				Any       =							prepare_image_inputs(self.image_processor_tester   ,						equal_resolution=__UpperCAmelCase   ,						torchify=__UpperCAmelCase )
            for image in image_inputs:
                   self.assertIsInstance(__UpperCAmelCase   ,						torch.Tensor )

            # Test not batched input
            _lowerCAmelCase     :				Optional[Any]       =							(
                (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"])
                * self.image_processor_tester.num_channels
            ) + 2

            for max_patch in self.image_processor_tester.max_patches:
                   # Test not batched input
                   _lowerCAmelCase     :				str       =							image_processor(
                       image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(1, max_patch, expected_hidden_dim)   ,						)

                   # Test batched
                   _lowerCAmelCase     :				Optional[int]       =							image_processor(
                       __UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)   ,						)
@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11     ,       reason="`Pix2StructImageProcessor` requires `torch>=1.11.0`."     ,       )
@require_torch
@require_vision
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_     ,       unittest.TestCase						):



     """simple docstring"""






     __magic_name__           =     PixaStructImageProcessor if is_vision_available() else None







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Dict       =							PixaStructImageProcessingTester(self   ,						num_channels=4 )
            _lowerCAmelCase     :				Tuple       =							3







     @property
     def a						(			self ):

            '''simple docstring'''

            return self.image_processor_tester.prepare_image_processor_dict()







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							self.image_processing_class(**self.image_processor_dict )
            self.assertTrue(hasattr(__UpperCAmelCase   ,						'do_normalize' ) )
            self.assertTrue(hasattr(__UpperCAmelCase   ,						'do_convert_rgb' ) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Dict       =							self.image_processing_class(**self.image_processor_dict )
            # create random PIL images
            _lowerCAmelCase     :				Union[str, Any]       =							prepare_image_inputs(self.image_processor_tester   ,						equal_resolution=__UpperCAmelCase )
            for image in image_inputs:
                   self.assertIsInstance(__UpperCAmelCase   ,						Image.Image )

            # Test not batched input
            _lowerCAmelCase     :				str       =							(
                (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"])
                * (self.image_processor_tester.num_channels - 1)
            ) + 2

            for max_patch in self.image_processor_tester.max_patches:
                   # Test not batched input
                   _lowerCAmelCase     :				List[str]       =							image_processor(
                       image_inputs[0]   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(1, max_patch, expected_hidden_dim)   ,						)

                   # Test batched
                   _lowerCAmelCase     :				int       =							image_processor(
                       __UpperCAmelCase   ,						return_tensors='pt'   ,						max_patches=__UpperCAmelCase ).flattened_patches
                   self.assertEqual(
                       encoded_images.shape   ,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)   ,						)






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}






def     SCREAMING_SNAKE_CASE(      __UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase )	->						Dict:
				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys() )}.' )

				if tokenizer_name is None:
								a__	:							str      						= TOKENIZER_CLASSES
				else:
								a__	:							int      						= {tokenizer_name: getattr(__UpperCamelCase	,				tokenizer_name + "Fast" )}

				logger.info(F'Loading tokenizer classes: {tokenizer_names}' )

				for tokenizer_name in tokenizer_names:
								a__	:							Any      						= TOKENIZER_CLASSES[tokenizer_name]

								a__	:							Dict      						= True
								if checkpoint_name is None:
												a__	:							Tuple      						= list(tokenizer_class.max_model_input_sizes.keys() )
								else:
												a__	:							List[str]      						= [checkpoint_name]

								logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}' )

								for checkpoint in checkpoint_names:
												logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}' )

												# Load tokenizer
												a__	:							List[str]      						= tokenizer_class.from_pretrained(__UpperCamelCase	,				force_download=__UpperCamelCase )

												# Save fast tokenizer
												logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}' )

												# For organization names we create sub-directories
												if "/" in checkpoint:
																a__						,   a__	:							Dict      						= checkpoint.split("/" )
																a__	:							Any      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
												elif add_prefix:
																a__	:							int      						= checkpoint
																a__	:							Optional[Any]      						= dump_path
												else:
																a__	:							Dict      						= None
																a__	:							Optional[Any]      						= dump_path

												logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values() )[0]:
																a__	:							Dict      						= list(tokenizer.pretrained_vocab_files_map.values() )[0][checkpoint]
																a__	:							Optional[Any]      						= file_path.split(__UpperCamelCase )[-1][0]
																if next_char == "/":
																				a__	:							Union[str, Any]      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
																				a__	:							Any      						= None

																logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												a__	:							Any      						= tokenizer.save_pretrained(
												    __UpperCamelCase	,				legacy_format=__UpperCamelCase	,				filename_prefix=__UpperCamelCase )
												logger.info(F'=> File names {file_names}' )

												for file_name in file_names:
																if not file_name.endswith("tokenizer.json" ):
																				os.remove(__UpperCamelCase )
																				logger.info(F'=> removing {file_name}' )


if __name__ == "__main__":
						lowerCamelCase		       =      argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
						)
						parser.add_argument(
						    """--tokenizer_name""",
						    default=None,
						    type=str,
						    help=(
						        F'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
						        """download and convert all the checkpoints from AWS."""
						    ),
						)
						parser.add_argument(
						    """--checkpoint_name""",
						    default=None,
						    type=str,
						    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
						)
						parser.add_argument(
						    """--force_download""",
						    action="""store_true""",
						    help="""Re-download checkpoints.""",
						)
						lowerCamelCase		       =      parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)


