
'''simple docstring'''
import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

__lowerCAmelCase												=	logging.get_logger(__name__)


__lowerCAmelCase												=	{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}."""				)

							if tokenizer_name is None:
														_snake_case       =  TOKENIZER_CLASSES
							else:
														_snake_case       =  {tokenizer_name: getattr(_SCREAMING_SNAKE_CASE							, tokenizer_name + """Fast"""				)}

							logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""				)

							for tokenizer_name in tokenizer_names:
														_snake_case       =  TOKENIZER_CLASSES[tokenizer_name]

														_snake_case       =  True
														if checkpoint_name is None:
																					_snake_case       =  list(tokenizer_class.max_model_input_sizes.keys()				)
														else:
																					_snake_case       =  [checkpoint_name]

														logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""				)

														for checkpoint in checkpoint_names:
																					logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""				)

																					# Load tokenizer
																					_snake_case       =  tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE							, force_download=_SCREAMING_SNAKE_CASE				)

																					# Save fast tokenizer
																					logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""				)

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												_snake_case,							_snake_case       =  checkpoint.split("""/"""				)
																												_snake_case       =  os.path.join(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
																					elif add_prefix:
																												_snake_case       =  checkpoint
																												_snake_case       =  dump_path
																					else:
																												_snake_case       =  None
																												_snake_case       =  dump_path

																					logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
																												_snake_case       =  list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
																												_snake_case       =  file_path.split(_SCREAMING_SNAKE_CASE				)[-1][0]
																												if next_char == "/":
																																			_snake_case       =  os.path.join(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
																																			_snake_case       =  None

																												logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

																					_snake_case       =  tokenizer.save_pretrained(
																					    _SCREAMING_SNAKE_CASE							, legacy_format=_SCREAMING_SNAKE_CASE							, filename_prefix=_SCREAMING_SNAKE_CASE				)
																					logger.info(f"""=> File names {file_names}"""				)

																					for file_name in file_names:
																												if not file_name.endswith("""tokenizer.json"""				):
																																			os.remove(_SCREAMING_SNAKE_CASE				)
																																			logger.info(f"""=> removing {file_name}"""				)


if __name__ == "__main__":
				__lowerCAmelCase												=	argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
				)
				parser.add_argument(
				    '--tokenizer_name',
				    default=None,
				    type=str,
				    help=(
				        f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
				        'download and convert all the checkpoints from AWS.'
				    ),
				)
				parser.add_argument(
				    '--checkpoint_name',
				    default=None,
				    type=str,
				    help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
				)
				parser.add_argument(
				    '--force_download',
				    action='store_true',
				    help='Re-download checkpoints.',
				)
				__lowerCAmelCase												=	parser.parse_args()

				convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


__lowerCAmelCase												=	logging.get_logger(__name__)

__lowerCAmelCase												=	{
    'shi-labs/nat-mini-in1k-224': 'https://huggingface.co/shi-labs/nat-mini-in1k-224/resolve/main/config.json',
    # See all Nat models at https://huggingface.co/models?filter=nat
}


class       _lowerCAmelCase   (   __snake_case					,							__snake_case				):







							'''simple docstring'''

							lowerCAmelCase_ 						=						"nat"

							lowerCAmelCase_ 						=						{
							    "num_attention_heads": "num_heads",
							    "num_hidden_layers": "num_layers",
							}

							def __init__(self  ,  UpperCAmelCase=4  ,  UpperCAmelCase=3  ,  UpperCAmelCase=64  ,  UpperCAmelCase=[3, 4, 6, 5]  ,  UpperCAmelCase=[2, 4, 8, 16]  ,  UpperCAmelCase=7  ,  UpperCAmelCase=3.0  ,  UpperCAmelCase=True  ,  UpperCAmelCase=0.0  ,  UpperCAmelCase=0.0  ,  UpperCAmelCase=0.1  ,  UpperCAmelCase="gelu"  ,  UpperCAmelCase=0.02  ,  UpperCAmelCase=1e-5  ,  UpperCAmelCase=0.0  ,  UpperCAmelCase=None  ,  UpperCAmelCase=None  ,  **UpperCAmelCase  ,  )					->       str:
														super().__init__(**UpperCAmelCase  )

														_snake_case       =  patch_size
														_snake_case       =  num_channels
														_snake_case       =  embed_dim
														_snake_case       =  depths
														_snake_case       =  len(UpperCAmelCase  )
														_snake_case       =  num_heads
														_snake_case       =  kernel_size
														_snake_case       =  mlp_ratio
														_snake_case       =  qkv_bias
														_snake_case       =  hidden_dropout_prob
														_snake_case       =  attention_probs_dropout_prob
														_snake_case       =  drop_path_rate
														_snake_case       =  hidden_act
														_snake_case       =  layer_norm_eps
														_snake_case       =  initializer_range
														# we set the hidden_size attribute in order to make Nat work with VisionEncoderDecoderModel
														# this indicates the channel dimension after the last stage of the model
														_snake_case       =  int(embed_dim * 2 ** (len(UpperCAmelCase  ) - 1)  )
														_snake_case       =  layer_scale_init_value
														_snake_case       =  ["""stem"""] + [f"""stage{idx}""" for idx in range(1  ,  len(UpperCAmelCase  ) + 1  )]
														_snake_case,							_snake_case       =  get_aligned_output_features_output_indices(
														    out_features=UpperCAmelCase  ,  out_indices=UpperCAmelCase  ,  stage_names=self.stage_names  )