




'''simple docstring'''



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import VivitImageProcessor






class 							_a				(			unittest.TestCase ):





       '''simple docstring'''






       def __init__(      self,							A,							A=7,							A=3,							A=10,							A=18,							A=30,							A=400,							A=True,							A=None,							A=True,							A=[0.5, 0.5, 0.5],							A=[0.5, 0.5, 0.5],							A=None,							):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						size if size is not None else {'shortest_edge': 18}
             SCREAMING_SNAKE_CASE   :      Dict    					=						crop_size if crop_size is not None else {'height': 18, 'width': 18}

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						parent
             SCREAMING_SNAKE_CASE   :      Dict    					=						batch_size
             SCREAMING_SNAKE_CASE   :      Dict    					=						num_channels
             SCREAMING_SNAKE_CASE   :      int    					=						num_frames
             SCREAMING_SNAKE_CASE   :      Dict    					=						image_size
             SCREAMING_SNAKE_CASE   :      Dict    					=						min_resolution
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						max_resolution
             SCREAMING_SNAKE_CASE   :      int    					=						do_resize
             SCREAMING_SNAKE_CASE   :      str    					=						size
             SCREAMING_SNAKE_CASE   :      str    					=						do_normalize
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						image_mean
             SCREAMING_SNAKE_CASE   :      Any    					=						image_std
             SCREAMING_SNAKE_CASE   :      Tuple    					=						crop_size







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             return {
                 "image_mean": self.image_mean,
                 "image_std": self.image_std,
                 "do_normalize": self.do_normalize,
                 "do_resize": self.do_resize,
                 "size": self.size,
                 "crop_size": self.crop_size,
             }






@require_torch
@require_vision
class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				Union[str, Any]        =						VivitImageProcessor if is_vision_available() else None







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						VivitImageProcessingTester(self  )







       @property
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             return self.image_processor_tester.prepare_image_processor_dict()







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Dict    					=						self.image_processing_class(**self.image_processor_dict  )
             self.assertTrue(hasattr(A,							'image_mean'  )  )
             self.assertTrue(hasattr(A,							'image_std'  )  )
             self.assertTrue(hasattr(A,							'do_normalize'  )  )
             self.assertTrue(hasattr(A,							'do_resize'  )  )
             self.assertTrue(hasattr(A,							'do_center_crop'  )  )
             self.assertTrue(hasattr(A,							'size'  )  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[str]    					=						self.image_processing_class.from_dict(self.image_processor_dict  )
             self.assertEqual(image_processor.size,							{'shortest_edge': 18}  )
             self.assertEqual(image_processor.crop_size,							{'height': 18, 'width': 18}  )

             SCREAMING_SNAKE_CASE   :      Any    					=						self.image_processing_class.from_dict(self.image_processor_dict,							size=42,							crop_size=84  )
             self.assertEqual(image_processor.size,							{'shortest_edge': 42}  )
             self.assertEqual(image_processor.crop_size,							{'height': 84, 'width': 84}  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						self.image_processing_class(**self.image_processor_dict  )
             # create random PIL videos
             SCREAMING_SNAKE_CASE   :      Tuple    					=						prepare_video_inputs(self.image_processor_tester,							equal_resolution=A  )
             for video in video_inputs:
                   self.assertIsInstance(A,							A  )
                   self.assertIsInstance(video[0],							Image.Image  )

             # Test not batched input
             SCREAMING_SNAKE_CASE   :      int    					=						image_processing(video_inputs[0],							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     1,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)

             # Test batched
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						image_processing(A,							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     self.image_processor_tester.batch_size,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      str    					=						self.image_processing_class(**self.image_processor_dict  )
             # create random numpy tensors
             SCREAMING_SNAKE_CASE   :      Any    					=						prepare_video_inputs(self.image_processor_tester,							equal_resolution=A,							numpify=A  )
             for video in video_inputs:
                   self.assertIsInstance(A,							A  )
                   self.assertIsInstance(video[0],							np.ndarray  )

             # Test not batched input
             SCREAMING_SNAKE_CASE   :      Any    					=						image_processing(video_inputs[0],							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     1,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)

             # Test batched
             SCREAMING_SNAKE_CASE   :      str    					=						image_processing(A,							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     self.image_processor_tester.batch_size,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.image_processing_class(**self.image_processor_dict  )
             # create random PyTorch tensors
             SCREAMING_SNAKE_CASE   :      Tuple    					=						prepare_video_inputs(self.image_processor_tester,							equal_resolution=A,							torchify=A  )
             for video in video_inputs:
                   self.assertIsInstance(A,							A  )
                   self.assertIsInstance(video[0],							torch.Tensor  )

             # Test not batched input
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						image_processing(video_inputs[0],							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     1,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)

             # Test batched
             SCREAMING_SNAKE_CASE   :      Dict    					=						image_processing(A,							return_tensors='pt'  ).pixel_values
             self.assertEqual(
                 encoded_videos.shape,							(
                     self.image_processor_tester.batch_size,
                     self.image_processor_tester.num_frames,
                     self.image_processor_tester.num_channels,
                     self.image_processor_tester.crop_size['height'],
                     self.image_processor_tester.crop_size['width'],
                 ),							)






'''simple docstring'''


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCamelCase 							=  logging.get_logger(__name__)


_UpperCamelCase 							=  {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(f"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()							)}."							)

     if tokenizer_name is None:
          __A	:       Tuple      =							TOKENIZER_CLASSES
     else:
          __A	:       Dict      =							{tokenizer_name: getattr(SCREAMING_SNAKE_CASE       ,					tokenizer_name + "Fast"							)}

     logger.info(f"Loading tokenizer classes: {tokenizer_names}"							)

     for tokenizer_name in tokenizer_names:
          __A	:       int      =							TOKENIZER_CLASSES[tokenizer_name]

          __A	:       Union[str, Any]      =							True
          if checkpoint_name is None:
               __A	:       Dict      =							list(tokenizer_class.max_model_input_sizes.keys()							)
          else:
               __A	:       str      =							[checkpoint_name]

          logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"							)

          for checkpoint in checkpoint_names:
               logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}"							)

               # Load tokenizer
               __A	:       List[str]      =							tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE       ,					force_download=SCREAMING_SNAKE_CASE							)

               # Save fast tokenizer
               logger.info(f"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"							)

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    __A  ,__A	:       Optional[Any]      =							checkpoint.split("/"							)
                    __A	:       str      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
               elif add_prefix:
                    __A	:       Optional[int]      =							checkpoint
                    __A	:       Any      =							dump_path
               else:
                    __A	:       int      =							None
                    __A	:       Optional[int]      =							dump_path

               logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()							)[0]:
                    __A	:       Any      =							list(tokenizer.pretrained_vocab_files_map.values()							)[0][checkpoint]
                    __A	:       Optional[int]      =							file_path.split(SCREAMING_SNAKE_CASE							)[-1][0]
                    if next_char == "/":
                         __A	:       Union[str, Any]      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                         __A	:       Union[str, Any]      =							None

                    logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               __A	:       Tuple      =							tokenizer.save_pretrained(
                   SCREAMING_SNAKE_CASE       ,					legacy_format=SCREAMING_SNAKE_CASE       ,					filename_prefix=SCREAMING_SNAKE_CASE							)
               logger.info(f"=> File names {file_names}"							)

               for file_name in file_names:
                    if not file_name.endswith("tokenizer.json"							):
                         os.remove(SCREAMING_SNAKE_CASE							)
                         logger.info(f"=> removing {file_name}"							)


if __name__ == "__main__":
      _UpperCamelCase 							=  argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      _UpperCamelCase 							=  parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

