'''simple docstring'''






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCAmelCase    :     List[str]         =       logging.get_logger(__name__)


_UpperCAmelCase    :     Any         =       {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Optional[int]   ,		__snake_case							:			Optional[Any]   ,		__snake_case							:			Optional[Any]   ,		__snake_case							:			Any				):
     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}.'				)

     if tokenizer_name is None:
          _A    		=  TOKENIZER_CLASSES
     else:
          _A    		=  {tokenizer_name: getattr(__snake_case   ,		tokenizer_name + 'Fast'				)}

     logger.info(F'Loading tokenizer classes: {tokenizer_names}'				)

     for tokenizer_name in tokenizer_names:
          _A    		=  TOKENIZER_CLASSES[tokenizer_name]

          _A    		=  True
          if checkpoint_name is None:
               _A    		=  list(tokenizer_class.max_model_input_sizes.keys()				)
          else:
               _A    		=  [checkpoint_name]

          logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'				)

          for checkpoint in checkpoint_names:
               logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'				)

               # Load tokenizer
               _A    		=  tokenizer_class.from_pretrained(__snake_case   ,		force_download=__snake_case				)

               # Save fast tokenizer
               logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'				)

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    _A		,       _A    		=  checkpoint.split('/'				)
                    _A    		=  os.path.join(__snake_case   ,		__snake_case				)
               elif add_prefix:
                    _A    		=  checkpoint
                    _A    		=  dump_path
               else:
                    _A    		=  None
                    _A    		=  dump_path

               logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
                    _A    		=  list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
                    _A    		=  file_path.split(__snake_case				)[-1][0]
                    if next_char == "/":
                         _A    		=  os.path.join(__snake_case   ,		__snake_case				)
                         _A    		=  None

                    logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'				)

               _A    		=  tokenizer.save_pretrained(
                   __snake_case   ,		legacy_format=__snake_case   ,		filename_prefix=__snake_case				)
               logger.info(F'=> File names {file_names}'				)

               for file_name in file_names:
                    if not file_name.endswith('tokenizer.json'				):
                         os.remove(__snake_case				)
                         logger.info(F'=> removing {file_name}'				)


if __name__ == "__main__":
   _UpperCAmelCase    :     int         =       argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
   )
   parser.add_argument(
       '''--tokenizer_name''',
       default=None,
       type=str,
       help=(
           F'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
           '''download and convert all the checkpoints from AWS.'''
       ),
   )
   parser.add_argument(
       '''--checkpoint_name''',
       default=None,
       type=str,
       help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
   )
   parser.add_argument(
       '''--force_download''',
       action='''store_true''',
       help='''Re-download checkpoints.''',
   )
   _UpperCAmelCase    :     int         =       parser.parse_args()

   convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)







'''simple docstring'''





from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.distilbert.modeling_tf_distilbert import (
					    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFDistilBertForMaskedLM,
					    TFDistilBertForMultipleChoice,
					    TFDistilBertForQuestionAnswering,
					    TFDistilBertForSequenceClassification,
					    TFDistilBertForTokenClassification,
					    TFDistilBertModel,
					)
class 						a__					:





			'''simple docstring'''

			def __init__(  self							:			str			,    lowerCAmelCase_							:			str			,    )      ->		Optional[int]:
						__A=	parent
						__A=	13
						__A=	7
						__A=	True
						__A=	True
						__A=	False
						__A=	True
						__A=	99
						__A=	32
						__A=	2
						__A=	4
						__A=	37
						__A=	'gelu'
						__A=	0.1
						__A=	0.1
						__A=	512
						__A=	16
						__A=	2
						__A=	0.02
						__A=	3
						__A=	4
						__A=	None
			def 					lowerCAmelCase     (  self							:			Optional[Any]	)      ->		str:
						__A=	ids_tensor([self.batch_size, self.seq_length]			,    self.vocab_size	)

						__A=	None
						if self.use_input_mask:
									__A=	random_attention_mask([self.batch_size, self.seq_length]	)

						__A=	None
						__A=	None
						__A=	None
						if self.use_labels:
									__A=	ids_tensor([self.batch_size]			,    self.type_sequence_label_size	)
									__A=	ids_tensor([self.batch_size, self.seq_length]			,    self.num_labels	)
									__A=	ids_tensor([self.batch_size]			,    self.num_choices	)

						__A=	DistilBertConfig(
						    vocab_size=self.vocab_size			,    dim=self.hidden_size			,    n_layers=self.num_hidden_layers			,    n_heads=self.num_attention_heads			,    hidden_dim=self.intermediate_size			,    hidden_act=self.hidden_act			,    dropout=self.hidden_dropout_prob			,    attention_dropout=self.attention_probs_dropout_prob			,    max_position_embeddings=self.max_position_embeddings			,    initializer_range=self.initializer_range			,    )

						return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Any			,    lowerCAmelCase_							:			int			,    lowerCAmelCase_							:			int			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			Any			,    lowerCAmelCase_							:			Optional[Any]	)      ->		Any:
						__A=	TFDistilBertModel(config=lowerCAmelCase_	)
						__A=	{'input_ids': input_ids, 'attention_mask': input_mask}

						__A=	model(lowerCAmelCase_	)

						__A=	[input_ids, input_mask]

						__A=	model(lowerCAmelCase_	)

						self.parent.assertEqual(result.last_hidden_state.shape			,    (self.batch_size, self.seq_length, self.hidden_size)	)
			def 					lowerCAmelCase     (  self							:			Optional[int]			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			List[Any]			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			Tuple	)      ->		Optional[int]:
						__A=	TFDistilBertForMaskedLM(config=lowerCAmelCase_	)
						__A=	{'input_ids': input_ids, 'attention_mask': input_mask}
						__A=	model(lowerCAmelCase_	)
						self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.seq_length, self.vocab_size)	)
			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			Tuple			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			List[str]	)      ->		int:
						__A=	TFDistilBertForQuestionAnswering(config=lowerCAmelCase_	)
						__A=	{
						    'input_ids': input_ids,
						    'attention_mask': input_mask,
						}
						__A=	model(lowerCAmelCase_	)
						self.parent.assertEqual(result.start_logits.shape			,    (self.batch_size, self.seq_length)	)
						self.parent.assertEqual(result.end_logits.shape			,    (self.batch_size, self.seq_length)	)
			def 					lowerCAmelCase     (  self							:			List[str]			,    lowerCAmelCase_							:			Any			,    lowerCAmelCase_							:			Optional[int]			,    lowerCAmelCase_							:			Tuple			,    lowerCAmelCase_							:			int			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			List[str]	)      ->		Union[str, Any]:
						__A=	self.num_labels
						__A=	TFDistilBertForSequenceClassification(lowerCAmelCase_	)
						__A=	{'input_ids': input_ids, 'attention_mask': input_mask}
						__A=	model(lowerCAmelCase_	)
						self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.num_labels)	)
			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Any			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			List[Any]			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			int	)      ->		Optional[Any]:
						__A=	self.num_choices
						__A=	TFDistilBertForMultipleChoice(lowerCAmelCase_	)
						__A=	tf.tile(tf.expand_dims(lowerCAmelCase_			,    1	)			,    (1, self.num_choices, 1)	)
						__A=	tf.tile(tf.expand_dims(lowerCAmelCase_			,    1	)			,    (1, self.num_choices, 1)	)
						__A=	{
						    'input_ids': multiple_choice_inputs_ids,
						    'attention_mask': multiple_choice_input_mask,
						}
						__A=	model(lowerCAmelCase_	)
						self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.num_choices)	)
			def 					lowerCAmelCase     (  self							:			List[Any]			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			Dict			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			Optional[int]			,    lowerCAmelCase_							:			List[str]			,    lowerCAmelCase_							:			Optional[Any]	)      ->		Dict:
						__A=	self.num_labels
						__A=	TFDistilBertForTokenClassification(lowerCAmelCase_	)
						__A=	{'input_ids': input_ids, 'attention_mask': input_mask}
						__A=	model(lowerCAmelCase_	)
						self.parent.assertEqual(result.logits.shape			,    (self.batch_size, self.seq_length, self.num_labels)	)

			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		int:
						__A=	self.prepare_config_and_inputs()
						((__A),	(__A),	(__A),	(__A),	(__A),	(__A))=	config_and_inputs
						__A=	{'input_ids': input_ids, 'attention_mask': input_mask}
						return config, inputs_dict


@require_tf
class 						a__					(					a_      ,  a_      ,  unittest.TestCase   ):





			'''simple docstring'''

			A						:				Optional[Any]					      =			(
			    (
			        TFDistilBertModel,
			        TFDistilBertForMaskedLM,
			        TFDistilBertForQuestionAnswering,
			        TFDistilBertForSequenceClassification,
			        TFDistilBertForTokenClassification,
			        TFDistilBertForMultipleChoice,
			    )
			    if is_tf_available()
			    else None
			)
			A						:				Optional[int]					      =			(
			    {
			        '''feature-extraction''': TFDistilBertModel,
			        '''fill-mask''': TFDistilBertForMaskedLM,
			        '''question-answering''': TFDistilBertForQuestionAnswering,
			        '''text-classification''': TFDistilBertForSequenceClassification,
			        '''token-classification''': TFDistilBertForTokenClassification,
			        '''zero-shot''': TFDistilBertForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			A						:				str					      =			False
			A						:				List[Any]					      =			False
			def 					lowerCAmelCase     (  self							:			List[str]	)      ->		Optional[Any]:
						__A=	TFDistilBertModelTester(self	)
						__A=	ConfigTester(self			,    config_class=lowerCAmelCase_			,    dim=37	)
			def 					lowerCAmelCase     (  self							:			Dict	)      ->		Tuple:
						self.config_tester.run_common_tests()
			def 					lowerCAmelCase     (  self							:			Union[str, Any]	)      ->		Optional[int]:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_model(*lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			Optional[int]	)      ->		Optional[Any]:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_masked_lm(*lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			str	)      ->		Any:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_question_answering(*lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			Tuple	)      ->		Optional[Any]:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			Optional[Any]	)      ->		Optional[Any]:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowerCAmelCase_	)
			def 					lowerCAmelCase     (  self							:			Any	)      ->		Optional[int]:
						__A=	self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_token_classification(*lowerCAmelCase_	)

			@slow
			def 					lowerCAmelCase     (  self							:			int	)      ->		Tuple:
						for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]	):
									__A=	TFDistilBertModel.from_pretrained(lowerCAmelCase_	)
									self.assertIsNotNone(lowerCAmelCase_	)


@require_tf
class 						a__					(					unittest.TestCase   ):





			'''simple docstring'''

			@slow
			def 					lowerCAmelCase     (  self							:			Any	)      ->		List[Any]:
						__A=	TFDistilBertModel.from_pretrained('distilbert-base-uncased'	)
						__A=	tf.constant([[0, 1, 2, 3, 4, 5]]	)
						__A=	model(lowerCAmelCase_	)[0]

						__A=	[1, 6, 768]
						self.assertEqual(output.shape			,    lowerCAmelCase_	)

						__A=	tf.constant(
						    [
						        [
						            [0.19_26_18_85, -0.13_73_29_55, 0.4_11_97_99],
						            [0.22_15_01_56, -0.07_42_26_61, 0.39_03_72_04],
						            [0.22_75_60_18, -0.0_89_64_14, 0.3_70_14_67],
						        ]
						    ]	)
						tf.debugging.assert_near(output[:, :3, :3]			,    lowerCAmelCase_			,    atol=1E-4	)



