


'''simple docstring'''
import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

UpperCamelCase__          = logging.get_logger(__name__)


UpperCamelCase__          = {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}





def    a__			(   lowerCAmelCase__    ,   lowerCAmelCase__    ,   lowerCAmelCase__    ,   lowerCAmelCase__		)       ->      Optional[int]:
    if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
        raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()		)}."""		)

    if tokenizer_name is None:
        UpperCAmelCase__     :						str						=							TOKENIZER_CLASSES
    else:
        UpperCAmelCase__     :						str						=							{tokenizer_name: getattr(lowerCAmelCase__    ,   tokenizer_name + '''Fast'''		)}

    logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""		)

    for tokenizer_name in tokenizer_names:
        UpperCAmelCase__     :						Tuple						=							TOKENIZER_CLASSES[tokenizer_name]

        UpperCAmelCase__     :						str						=							True
        if checkpoint_name is None:
            UpperCAmelCase__     :						str						=							list(tokenizer_class.max_model_input_sizes.keys()		)
        else:
            UpperCAmelCase__     :						Tuple						=							[checkpoint_name]

        logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""		)

        for checkpoint in checkpoint_names:
            logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""		)

            # Load tokenizer
            UpperCAmelCase__     :						Dict						=							tokenizer_class.from_pretrained(lowerCAmelCase__    ,   force_download=lowerCAmelCase__		)

            # Save fast tokenizer
            logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""		)

            # For organization names we create sub-directories
            if "/" in checkpoint:
                UpperCAmelCase__		,		UpperCAmelCase__     :						Dict						=							checkpoint.split('''/'''		)
                UpperCAmelCase__     :						List[Any]						=							os.path.join(lowerCAmelCase__    ,   lowerCAmelCase__		)
            elif add_prefix:
                UpperCAmelCase__     :						Tuple						=							checkpoint
                UpperCAmelCase__     :						Any						=							dump_path
            else:
                UpperCAmelCase__     :						Optional[int]						=							None
                UpperCAmelCase__     :						Dict						=							dump_path

            logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""		)

            if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()		)[0]:
                UpperCAmelCase__     :						Union[str, Any]						=							list(tokenizer.pretrained_vocab_files_map.values()		)[0][checkpoint]
                UpperCAmelCase__     :						Optional[int]						=							file_path.split(lowerCAmelCase__		)[-1][0]
                if next_char == "/":
                    UpperCAmelCase__     :						int						=							os.path.join(lowerCAmelCase__    ,   lowerCAmelCase__		)
                    UpperCAmelCase__     :						List[Any]						=							None

                logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""		)

            UpperCAmelCase__     :						Optional[Any]						=							tokenizer.save_pretrained(
                lowerCAmelCase__    ,   legacy_format=lowerCAmelCase__    ,   filename_prefix=lowerCAmelCase__		)
            logger.info(F"""=> File names {file_names}"""		)

            for file_name in file_names:
                if not file_name.endswith('''tokenizer.json'''		):
                    os.remove(lowerCAmelCase__		)
                    logger.info(F"""=> removing {file_name}"""		)


if __name__ == "__main__":
  UpperCamelCase__          = argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
  )
  parser.add_argument(
      '''--tokenizer_name''',
      default=None,
      type=str,
      help=(
          F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
          '''download and convert all the checkpoints from AWS.'''
      ),
  )
  parser.add_argument(
      '''--checkpoint_name''',
      default=None,
      type=str,
      help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
  )
  parser.add_argument(
      '''--force_download''',
      action='''store_true''',
      help='''Re-download checkpoints.''',
  )
  UpperCamelCase__          = parser.parse_args()

  convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)






'''simple docstring'''
import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow
@is_pipeline_test
@require_torch
class   __UpperCAmelCase				(     unittest.TestCase				):



					'''simple docstring'''





					@require_torch
					def     __snake_case						(	self     :						str)    ->							Optional[int]:
								A_									= pipeline(
								    task='zero-shot-audio-classification'      ,       model='hf-internal-testing/tiny-clap-htsat-unfused')
								A_									= load_dataset('ashraq/esc50')
								A_									= dataset['train']['audio'][-1]['array']
								A_									= audio_classifier(_lowercase      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [{'score': 0.5_01, 'label': 'Sound of a dog'}, {'score': 0.4_99, 'label': 'Sound of vaccum cleaner'}]      ,       )



					@unittest.skip('No models are available in TF')
					def     __snake_case						(	self     :						Tuple)    ->							str:
								pass



					@slow
					@require_torch
					def     __snake_case						(	self     :						Optional[Any])    ->							Tuple:
								A_									= pipeline(
								    task='zero-shot-audio-classification'      ,       model='laion/clap-htsat-unfused'      ,       )
								# This is an audio of a dog
								A_									= load_dataset('ashraq/esc50')
								A_									= dataset['train']['audio'][-1]['array']
								A_									= audio_classifier(_lowercase      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])

								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        {'score': 0.9_99, 'label': 'Sound of a dog'},
								        {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								    ]      ,       )

								A_									= audio_classifier([audio] * 5      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        [
								            {'score': 0.9_99, 'label': 'Sound of a dog'},
								            {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								        ],
								    ]
								    * 5      ,       )
								A_									= audio_classifier(
								    [audio] * 5      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner']      ,       batch_size=5)
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        [
								            {'score': 0.9_99, 'label': 'Sound of a dog'},
								            {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								        ],
								    ]
								    * 5      ,       )



					@unittest.skip('No models are available in TF')
					def     __snake_case						(	self     :						List[str])    ->							str:
								pass




