



"""simple docstring"""






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__snake_case				:  Optional[Any] =       logging.get_logger(__name__)


__snake_case				:  List[Any] =       {name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}


def      _lowercase							(    __snake_case   ,__snake_case   ,__snake_case   ,__snake_case      )		->  Optional[int]:
			if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
						raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()      )}."""      )

			if tokenizer_name is None:
						__lowerCAmelCase :    Dict				=			TOKENIZER_CLASSES
			else:
						__lowerCAmelCase :    Optional[int]				=			{tokenizer_name: getattr(__snake_case   ,tokenizer_name + "Fast"      )}

			logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""      )

			for tokenizer_name in tokenizer_names:
						__lowerCAmelCase :    Union[str, Any]				=			TOKENIZER_CLASSES[tokenizer_name]

						__lowerCAmelCase :    Dict				=			True
						if checkpoint_name is None:
									__lowerCAmelCase :    Optional[Any]				=			list(tokenizer_class.max_model_input_sizes.keys()      )
						else:
									__lowerCAmelCase :    Any				=			[checkpoint_name]

						logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""      )

						for checkpoint in checkpoint_names:
									logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""      )

									# Load tokenizer
									__lowerCAmelCase :    Optional[Any]				=			tokenizer_class.from_pretrained(__snake_case   ,force_download=__snake_case      )

									# Save fast tokenizer
									logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""      )

									# For organization names we create sub-directories
									if "/" in checkpoint:
												__lowerCAmelCase		,      __lowerCAmelCase :    Tuple				=			checkpoint.split("/"      )
												__lowerCAmelCase :    int				=			os.path.join(__snake_case   ,__snake_case      )
									elif add_prefix:
												__lowerCAmelCase :    Tuple				=			checkpoint
												__lowerCAmelCase :    str				=			dump_path
									else:
												__lowerCAmelCase :    Dict				=			None
												__lowerCAmelCase :    Dict				=			dump_path

									logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

									if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()      )[0]:
												__lowerCAmelCase :    Optional[int]				=			list(tokenizer.pretrained_vocab_files_map.values()      )[0][checkpoint]
												__lowerCAmelCase :    Dict				=			file_path.split(__snake_case      )[-1][0]
												if next_char == "/":
															__lowerCAmelCase :    List[str]				=			os.path.join(__snake_case   ,__snake_case      )
															__lowerCAmelCase :    Optional[int]				=			None

												logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

									__lowerCAmelCase :    Dict				=			tokenizer.save_pretrained(
									    __snake_case   ,legacy_format=__snake_case   ,filename_prefix=__snake_case      )
									logger.info(F"""=> File names {file_names}"""      )

									for file_name in file_names:
												if not file_name.endswith("tokenizer.json"      ):
															os.remove(__snake_case      )
															logger.info(F"""=> removing {file_name}"""      )


if __name__ == "__main__":
	__snake_case				:  List[Any] =       argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
	)
	parser.add_argument(
	    '--tokenizer_name',
	    default=None,
	    type=str,
	    help=(
	        F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
	        'download and convert all the checkpoints from AWS.'
	    ),
	)
	parser.add_argument(
	    '--checkpoint_name',
	    default=None,
	    type=str,
	    help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
	)
	parser.add_argument(
	    '--force_download',
	    action='store_true',
	    help='Re-download checkpoints.',
	)
	__snake_case				:  Optional[int] =       parser.parse_args()

	convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




"""simple docstring"""






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:  str =       logging.get_logger(__name__)

__snake_case				:  Union[str, Any] =       {
    'microsoft/git-base': 'https://huggingface.co/microsoft/git-base/resolve/main/config.json',
}



class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'git_vision_model'

	def __init__(	self:						Optional[int]			,     _SCREAMING_SNAKE_CASE:						List[Any]=768			,     _SCREAMING_SNAKE_CASE:						Optional[Any]=3072			,     _SCREAMING_SNAKE_CASE:						str=12			,     _SCREAMING_SNAKE_CASE:						Any=12			,     _SCREAMING_SNAKE_CASE:						Dict=3			,     _SCREAMING_SNAKE_CASE:						List[Any]=224			,     _SCREAMING_SNAKE_CASE:						List[str]=16			,     _SCREAMING_SNAKE_CASE:						str="quick_gelu"			,     _SCREAMING_SNAKE_CASE:						Any=1e-5			,     _SCREAMING_SNAKE_CASE:						List[str]=0.0			,     _SCREAMING_SNAKE_CASE:						Dict=0.02			,     **_SCREAMING_SNAKE_CASE:						str			,     )							->			Optional[Any]:

				"""simple docstring"""

				super().__init__(**_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    Union[str, Any]				=			hidden_size
				__lowerCAmelCase :    List[str]				=			intermediate_size
				__lowerCAmelCase :    List[str]				=			num_hidden_layers
				__lowerCAmelCase :    Tuple				=			num_attention_heads
				__lowerCAmelCase :    List[Any]				=			num_channels
				__lowerCAmelCase :    Dict				=			patch_size
				__lowerCAmelCase :    Optional[int]				=			image_size
				__lowerCAmelCase :    Optional[Any]				=			initializer_range
				__lowerCAmelCase :    str				=			attention_dropout
				__lowerCAmelCase :    Optional[Any]				=			layer_norm_eps
				__lowerCAmelCase :    Tuple				=			hidden_act







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Dict			,     _SCREAMING_SNAKE_CASE:						Union[str, os.PathLike]			,     **_SCREAMING_SNAKE_CASE:						Dict)							->			"PretrainedConfig":

				"""simple docstring"""

				cls._set_token_in_kwargs(_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase		,      __lowerCAmelCase :    List[Any]				=			cls.get_config_dict(_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				# get the vision config dict if we are loading from GITConfig
				if config_dict.get("model_type") == "git":
							__lowerCAmelCase :    str				=			config_dict["vision_config"]

				if "model_type" in config_dict and hasattr(cls			,     "model_type") and config_dict["model_type"] != cls.model_type:
							logger.warning(
							    F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
							    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors.""")

				return cls.from_dict(_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)






class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'git'

	def __init__(	self:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						Tuple=None			,     _SCREAMING_SNAKE_CASE:						Dict=3_0522			,     _SCREAMING_SNAKE_CASE:						Any=768			,     _SCREAMING_SNAKE_CASE:						str=6			,     _SCREAMING_SNAKE_CASE:						int=12			,     _SCREAMING_SNAKE_CASE:						Dict=3072			,     _SCREAMING_SNAKE_CASE:						Optional[Any]="gelu"			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=0.1			,     _SCREAMING_SNAKE_CASE:						Any=0.1			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=1024			,     _SCREAMING_SNAKE_CASE:						str=0.02			,     _SCREAMING_SNAKE_CASE:						Tuple=1e-12			,     _SCREAMING_SNAKE_CASE:						List[str]=0			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]="absolute"			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=True			,     _SCREAMING_SNAKE_CASE:						Tuple=False			,     _SCREAMING_SNAKE_CASE:						Dict=101			,     _SCREAMING_SNAKE_CASE:						Tuple=102			,     _SCREAMING_SNAKE_CASE:						Any=None			,     **_SCREAMING_SNAKE_CASE:						List[Any]			,     )							->			List[Any]:

				"""simple docstring"""

				super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE			,     eos_token_id=_SCREAMING_SNAKE_CASE			,     pad_token_id=_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				if vision_config is None:
							__lowerCAmelCase :    int				=			{}
							logger.info("vision_config is None. initializing the GitVisionConfig with default values.")

				__lowerCAmelCase :    List[str]				=			GitVisionConfig(**_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    List[Any]				=			vocab_size
				__lowerCAmelCase :    int				=			hidden_size
				__lowerCAmelCase :    Dict				=			num_hidden_layers
				__lowerCAmelCase :    Tuple				=			num_attention_heads
				__lowerCAmelCase :    Union[str, Any]				=			hidden_act
				__lowerCAmelCase :    Dict				=			intermediate_size
				__lowerCAmelCase :    List[str]				=			hidden_dropout_prob
				__lowerCAmelCase :    Union[str, Any]				=			attention_probs_dropout_prob
				__lowerCAmelCase :    Union[str, Any]				=			max_position_embeddings
				__lowerCAmelCase :    Optional[int]				=			initializer_range
				__lowerCAmelCase :    str				=			layer_norm_eps
				__lowerCAmelCase :    Optional[Any]				=			position_embedding_type
				__lowerCAmelCase :    str				=			use_cache
				__lowerCAmelCase :    str				=			tie_word_embeddings
				__lowerCAmelCase :    Optional[int]				=			num_image_with_embedding

				__lowerCAmelCase :    Optional[Any]				=			bos_token_id
				__lowerCAmelCase :    Any				=			eos_token_id







	def 				_SCREAMING_SNAKE_CASE  (	self:						Dict)							->			List[str]:

				"""simple docstring"""

				__lowerCAmelCase :    Optional[int]				=			copy.deepcopy(self.__dict__)
				__lowerCAmelCase :    List[Any]				=			self.vision_config.to_dict()
				__lowerCAmelCase :    Tuple				=			self.__class__.model_type
				return output