




'''simple docstring'''





import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)


UpperCamelCase_		:     Optional[int]          =		{name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}





def       __a      (  _UpperCamelCase:       Tuple   ,		_UpperCamelCase:       Tuple   ,		_UpperCamelCase:       Optional[int]   ,		_UpperCamelCase:       Union[str, Any]       )		->					Tuple:

     """simple docstring"""




     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()       )}."""       )

     if tokenizer_name is None:
          _snake_case				= TOKENIZER_CLASSES
     else:
          _snake_case				= {tokenizer_name: getattr(_UpperCamelCase   ,		tokenizer_name + "Fast"       )}

     logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""       )

     for tokenizer_name in tokenizer_names:
          _snake_case				= TOKENIZER_CLASSES[tokenizer_name]

          _snake_case				= True
          if checkpoint_name is None:
               _snake_case				= list(tokenizer_class.max_model_input_sizes.keys()       )
          else:
               _snake_case				= [checkpoint_name]

          logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""       )

          for checkpoint in checkpoint_names:
               logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""       )

               # Load tokenizer
               _snake_case				= tokenizer_class.from_pretrained(_UpperCamelCase   ,		force_download=_UpperCamelCase       )

               # Save fast tokenizer
               logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""       )

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    _snake_case				,   _snake_case				= checkpoint.split("/"       )
                    _snake_case				= os.path.join(_UpperCamelCase   ,		_UpperCamelCase       )
               elif add_prefix:
                    _snake_case				= checkpoint
                    _snake_case				= dump_path
               else:
                    _snake_case				= None
                    _snake_case				= dump_path

               logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""       )

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()       )[0]:
                    _snake_case				= list(tokenizer.pretrained_vocab_files_map.values()       )[0][checkpoint]
                    _snake_case				= file_path.split(_UpperCamelCase       )[-1][0]
                    if next_char == "/":
                         _snake_case				= os.path.join(_UpperCamelCase   ,		_UpperCamelCase       )
                         _snake_case				= None

                    logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""       )

               _snake_case				= tokenizer.save_pretrained(
                   _UpperCamelCase   ,		legacy_format=_UpperCamelCase   ,		filename_prefix=_UpperCamelCase       )
               logger.info(F"""=> File names {file_names}"""       )

               for file_name in file_names:
                    if not file_name.endswith("tokenizer.json"       ):
                         os.remove(_UpperCamelCase       )
                         logger.info(F"""=> removing {file_name}"""       )


if __name__ == "__main__":
   UpperCamelCase_		:     Optional[Any]          =		argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
   )
   parser.add_argument(
       '''--tokenizer_name''',
       default=None,
       type=str,
       help=(
           F'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
           '''download and convert all the checkpoints from AWS.'''
       ),
   )
   parser.add_argument(
       '''--checkpoint_name''',
       default=None,
       type=str,
       help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
   )
   parser.add_argument(
       '''--force_download''',
       action='''store_true''',
       help='''Re-download checkpoints.''',
   )
   UpperCamelCase_		:     Union[str, Any]          =		parser.parse_args()

   convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)








'''simple docstring'''





import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
UpperCamelCase_		:     Optional[int]          =		object()

# For specifying empty leaf dict `{}`
UpperCamelCase_		:     Optional[int]          =		object()





def       __a      (  _UpperCamelCase:       Union[str, Any]   ,		_UpperCamelCase:       Optional[Any]       )		->					Optional[Any]:

     """simple docstring"""




     _snake_case				= tuple((re.compile(x + "$"       ) for x in qs)       )
     for i in range(len(_UpperCamelCase       ) - len(_UpperCamelCase       ) + 1       ):
          _snake_case				= [x.match(_UpperCamelCase       ) for x, y in zip(_UpperCamelCase   ,		ks[i:]       )]
          if matches and all(_UpperCamelCase       ):
               return True
     return False





def       __a      (  _UpperCamelCase:       Optional[Any]       )		->					Union[str, Any]:

     """simple docstring"""




     def replace(_UpperCamelCase:       Tuple   ,		_UpperCamelCase:       List[str]       ):
          for rule, replacement in rules:
               if _match(_UpperCamelCase   ,		_UpperCamelCase       ):
                    return replacement
          return val

     return replace





def       __a      (  )		->					Any:

     """simple docstring"""




     return [
         # embeddings
         (("transformer", "wpe", "embedding"), P("mp"   ,		_UpperCamelCase       )),
         (("transformer", "wte", "embedding"), P("mp"   ,		_UpperCamelCase       )),
         # atention
         (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(_UpperCamelCase   ,		"mp"       )),
         (("attention", "out_proj", "kernel"), P("mp"   ,		_UpperCamelCase       )),
         (("attention", "out_proj", "bias"), None),
         # mlp
         (("mlp", "c_fc", "kernel"), P(_UpperCamelCase   ,		"mp"       )),
         (("mlp", "c_fc", "bias"), P("mp"       )),
         (("mlp", "c_proj", "kernel"), P("mp"   ,		_UpperCamelCase       )),
         (("mlp", "c_proj", "bias"), None),
         # layer norms
         ((r"ln_\d+", "bias"), None),
         ((r"\d+", r"ln_\d+", "scale"), None),
         (("ln_f", "bias"), None),
         (("ln_f", "scale"), None),
     ]





def       __a      (  _UpperCamelCase:       Union[str, Any]       )		->					Any:

     """simple docstring"""




     _snake_case				= _get_partition_rules()
     _snake_case				= _replacement_rules(_UpperCamelCase       )
     _snake_case				= {k: _unmatched for k in flatten_dict(_UpperCamelCase       )}
     _snake_case				= {k: replace(_UpperCamelCase   ,		_UpperCamelCase       ) for k, v in initd.items()}
     assert _unmatched not in result.values(), "Incomplete partition spec."
     return freeze(unflatten_dict(_UpperCamelCase       )       )


