


'''simple docstring'''



import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

A__						:   Union[str, Any]				=    logging.get_logger(__name__)


A__						:   Union[str, Any]				=    {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Union[str, Any] ,       UpperCAmelCase_       :							List[str] ,       UpperCAmelCase_       :							Dict ,       UpperCAmelCase_       :							Any				)	->						Union[str, Any]:
     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}.'				)

     if tokenizer_name is None:
          __lowerCamelCase  :      List[str]														=						TOKENIZER_CLASSES
     else:
          __lowerCamelCase  :      List[Any]														=						{tokenizer_name: getattr(UpperCAmelCase_ ,       tokenizer_name + 'Fast'				)}

     logger.info(F'Loading tokenizer classes: {tokenizer_names}'				)

     for tokenizer_name in tokenizer_names:
          __lowerCamelCase  :      Dict														=						TOKENIZER_CLASSES[tokenizer_name]

          __lowerCamelCase  :      Optional[Any]														=						True
          if checkpoint_name is None:
               __lowerCamelCase  :      Dict														=						list(tokenizer_class.max_model_input_sizes.keys()				)
          else:
               __lowerCamelCase  :      int														=						[checkpoint_name]

          logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'				)

          for checkpoint in checkpoint_names:
               logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'				)

               # Load tokenizer
               __lowerCamelCase  :      Any														=						tokenizer_class.from_pretrained(UpperCAmelCase_ ,       force_download=UpperCAmelCase_				)

               # Save fast tokenizer
               logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'				)

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						checkpoint.split('/'				)
                    __lowerCamelCase  :      List[Any]														=						os.path.join(UpperCAmelCase_ ,       UpperCAmelCase_				)
               elif add_prefix:
                    __lowerCamelCase  :      int														=						checkpoint
                    __lowerCamelCase  :      Tuple														=						dump_path
               else:
                    __lowerCamelCase  :      List[str]														=						None
                    __lowerCamelCase  :      str														=						dump_path

               logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
                    __lowerCamelCase  :      Any														=						list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
                    __lowerCamelCase  :      List[Any]														=						file_path.split(UpperCAmelCase_				)[-1][0]
                    if next_char == "/":
                         __lowerCamelCase  :      Tuple														=						os.path.join(UpperCAmelCase_ ,       UpperCAmelCase_				)
                         __lowerCamelCase  :      str														=						None

                    logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

               __lowerCamelCase  :      Union[str, Any]														=						tokenizer.save_pretrained(
                   UpperCAmelCase_ ,       legacy_format=UpperCAmelCase_ ,       filename_prefix=UpperCAmelCase_				)
               logger.info(F'=> File names {file_names}'				)

               for file_name in file_names:
                    if not file_name.endswith('tokenizer.json'				):
                         os.remove(UpperCAmelCase_				)
                         logger.info(F'=> removing {file_name}'				)


if __name__ == "__main__":
      A__						:   int				=    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      A__						:   List[str]				=    parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)










'''simple docstring'''



import inspect
import unittest

import numpy as np

from transformers import ViTConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
      import jax

      from transformers.models.vit.modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel





class   UpperCAmelCase_    (unittest.TestCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=13     ,	SCREAMING_SNAKE_CASE_=30     ,	SCREAMING_SNAKE_CASE_=2     ,	SCREAMING_SNAKE_CASE_=3     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_=5     ,	SCREAMING_SNAKE_CASE_=4     ,	SCREAMING_SNAKE_CASE_=37     ,	SCREAMING_SNAKE_CASE_="gelu"     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=10     ,	SCREAMING_SNAKE_CASE_=0.0_2     ,	)							->	Optional[int]:
         __lowerCamelCase  :      Optional[int]														=						parent
         __lowerCamelCase  :      Dict														=						batch_size
         __lowerCamelCase  :      int														=						image_size
         __lowerCamelCase  :      List[str]														=						patch_size
         __lowerCamelCase  :      Optional[int]														=						num_channels
         __lowerCamelCase  :      Any														=						is_training
         __lowerCamelCase  :      Dict														=						use_labels
         __lowerCamelCase  :      List[Any]														=						hidden_size
         __lowerCamelCase  :      List[Any]														=						num_hidden_layers
         __lowerCamelCase  :      Optional[Any]														=						num_attention_heads
         __lowerCamelCase  :      Dict														=						intermediate_size
         __lowerCamelCase  :      Union[str, Any]														=						hidden_act
         __lowerCamelCase  :      Optional[int]														=						hidden_dropout_prob
         __lowerCamelCase  :      Tuple														=						attention_probs_dropout_prob
         __lowerCamelCase  :      str														=						type_sequence_label_size
         __lowerCamelCase  :      List[str]														=						initializer_range

         # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
         __lowerCamelCase  :      str														=						(image_size // patch_size) ** 2
         __lowerCamelCase  :      Optional[int]														=						num_patches + 1







    def  lowercase_							(				self      )							->	Union[str, Any]:
         __lowerCamelCase  :      int														=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

         __lowerCamelCase  :      Optional[int]														=						ViTConfig(
             image_size=self.image_size     ,	patch_size=self.patch_size     ,	num_channels=self.num_channels     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	is_decoder=SCREAMING_SNAKE_CASE_     ,	initializer_range=self.initializer_range     ,	)

         return config, pixel_values







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Union[str, Any]:
         __lowerCamelCase  :      Union[str, Any]														=						FlaxViTModel(config=SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      str														=						model(SCREAMING_SNAKE_CASE_      )
         # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
         __lowerCamelCase  :      str														=						(self.image_size, self.image_size)
         __lowerCamelCase  :      str														=						(self.patch_size, self.patch_size)
         __lowerCamelCase  :      Any														=						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
         self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, num_patches + 1, self.hidden_size)      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	str:
         __lowerCamelCase  :      Tuple														=						self.type_sequence_label_size
         __lowerCamelCase  :      Any														=						FlaxViTForImageClassification(config=SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Dict														=						model(SCREAMING_SNAKE_CASE_      )
         self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.type_sequence_label_size)      )

         # test greyscale images
         __lowerCamelCase  :      List[str]														=						1
         __lowerCamelCase  :      List[Any]														=						FlaxViTForImageClassification(SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      str														=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]      )
         __lowerCamelCase  :      List[Any]														=						model(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Optional[int]:
         __lowerCamelCase  :      List[Any]														=						self.prepare_config_and_inputs()
         (
             (
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			
         )  :      int														=						config_and_inputs
         __lowerCamelCase  :      Union[str, Any]														=						{'pixel_values': pixel_values}
         return config, inputs_dict







@require_flax
class   UpperCAmelCase_    (_UpperCAmelCase   ,						unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   str					=     (FlaxViTModel, FlaxViTForImageClassification) if is_flax_available() else ()







    def  lowercase_							(				self      )							->	None:
         __lowerCamelCase  :      str														=						FlaxViTModelTester(self      )
         __lowerCamelCase  :      Union[str, Any]														=						ConfigTester(self     ,	config_class=SCREAMING_SNAKE_CASE_     ,	has_text_modality=SCREAMING_SNAKE_CASE_     ,	hidden_size=37      )







    def  lowercase_							(				self      )							->	List[Any]:
         self.config_tester.run_common_tests()







    def  lowercase_							(				self      )							->	Optional[Any]:
         __lowerCamelCase  :      Optional[int]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      List[Any]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Optional[Any]:
         __lowerCamelCase							,			__lowerCamelCase  :      Optional[int]														=						self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              __lowerCamelCase  :      Optional[Any]														=						model_class(SCREAMING_SNAKE_CASE_      )
              __lowerCamelCase  :      Dict														=						inspect.signature(model.__call__      )
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              __lowerCamelCase  :      List[str]														=						[*signature.parameters.keys()]

              __lowerCamelCase  :      Optional[int]														=						['pixel_values']
              self.assertListEqual(arg_names[:1]     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              with self.subTest(model_class.__name__      ):
                   __lowerCamelCase  :      List[Any]														=						self._prepare_for_class(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Optional[int]														=						model_class(SCREAMING_SNAKE_CASE_      )

                   @jax.jit
                   def model_jitted(SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      ):
                        return model(pixel_values=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )

                   with self.subTest('JIT Enabled'      ):
                        __lowerCamelCase  :      Optional[int]														=						model_jitted(**SCREAMING_SNAKE_CASE_      ).to_tuple()

                   with self.subTest('JIT Disabled'      ):
                        with jax.disable_jit():
                             __lowerCamelCase  :      Union[str, Any]														=						model_jitted(**SCREAMING_SNAKE_CASE_      ).to_tuple()

                   self.assertEqual(len(SCREAMING_SNAKE_CASE_      )     ,	len(SCREAMING_SNAKE_CASE_      )      )
                   for jitted_output, output in zip(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ):
                        self.assertEqual(jitted_output.shape     ,	output.shape      )







    @slow
    def  lowercase_							(				self      )							->	List[Any]:
         for model_class_name in self.all_model_classes:
              __lowerCamelCase  :      Union[str, Any]														=						model_class_name.from_pretrained('google/vit-base-patch16-224'      )
              __lowerCamelCase  :      Union[str, Any]														=						model(np.ones((1, 3, 2_24, 2_24)      )      )
              self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )






