


'''simple docstring'''


from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
      from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


_UpperCamelCase 							=  logging.get_logger(__name__)

_UpperCamelCase 							=  {
    """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/resolve/main/config.json""",
    """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/config.json""",
    """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json""",
    """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json""",
    """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/config.json""",
    """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json""",
}







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				'bloom'
     lowerCamelCase__       						=				['past_key_values']
     lowerCamelCase__       						=				{
         'num_hidden_layers': 'n_layer',
         'num_attention_heads': 'n_head',
     }


     def __init__(   self			,     lowerCamelCase=25_0880			,     lowerCamelCase=64			,     lowerCamelCase=2			,     lowerCamelCase=8			,     lowerCamelCase=1E-5			,     lowerCamelCase=0.02			,     lowerCamelCase=True			,     lowerCamelCase=1			,     lowerCamelCase=2			,     lowerCamelCase=False			,     lowerCamelCase=0.0			,     lowerCamelCase=0.0			,     lowerCamelCase=1			,     lowerCamelCase=False			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          __A	:       List[str]      =							vocab_size
          # Backward compatibility with n_embed kwarg
          __A	:       List[Any]      =							kwargs.pop("n_embed"			,     lowerCamelCase		)
          __A	:       int      =							hidden_size if n_embed is None else n_embed
          __A	:       Optional[Any]      =							n_layer
          __A	:       Union[str, Any]      =							n_head
          __A	:       str      =							layer_norm_epsilon
          __A	:       Union[str, Any]      =							initializer_range
          __A	:       Union[str, Any]      =							use_cache
          __A	:       Dict      =							pretraining_tp
          __A	:       Optional[Any]      =							apply_residual_connection_post_layernorm
          __A	:       Optional[Any]      =							hidden_dropout
          __A	:       Union[str, Any]      =							attention_dropout

          __A	:       List[str]      =							bos_token_id
          __A	:       Tuple      =							eos_token_id
          __A	:       Any      =							slow_but_exact

          super().__init__(bos_token_id=lowerCamelCase			,     eos_token_id=lowerCamelCase			,     **lowerCamelCase		)







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				version.parse('1.12' )


     def __init__(   self			,     lowerCamelCase			,     lowerCamelCase = "default"			,     lowerCamelCase = None			,     lowerCamelCase = False			,     ):


          '''simple docstring'''





          super().__init__(lowerCamelCase			,     task=lowerCamelCase			,     patching_specs=lowerCamelCase			,     use_past=lowerCamelCase		)
          if not getattr(self._config			,     "pad_token_id"			,     lowerCamelCase		):
               # TODO: how to do that better?
               __A	:       Dict      =							0


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[int]      =							OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}		)
          if self.use_past:
               # BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
               self.fill_with_past_key_values_(lowerCamelCase			,     direction="inputs"			,     inverted_values_shape=lowerCamelCase		)
               __A	:       Any      =							{0: "batch", 1: "past_sequence + sequence"}
          else:
               __A	:       Tuple      =							{0: "batch", 1: "sequence"}

          return common_inputs


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return self._config.n_layer


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return self._config.n_head


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return 1E-3


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = -1			,     lowerCamelCase = -1			,     lowerCamelCase = False			,     lowerCamelCase = None			,     ):


          '''simple docstring'''





          __A	:       Union[str, Any]      =							super(lowerCamelCase			,     self		).generate_dummy_inputs(
              lowerCamelCase			,     batch_size=lowerCamelCase			,     seq_length=lowerCamelCase			,     is_pair=lowerCamelCase			,     framework=lowerCamelCase		)

          # We need to order the input in the way they appears in the forward()
          __A	:       Optional[Any]      =							OrderedDict({"input_ids": common_inputs["input_ids"]}		)

          # Need to add the past_keys
          if self.use_past:
               if not is_torch_available():
                    raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."		)
               else:
                    import torch

                    __A  ,__A	:       List[str]      =							common_inputs["input_ids"].shape
                    # Not using the same length for past_key_values
                    __A	:       Optional[int]      =							seqlen + 2
                    __A	:       Optional[int]      =							self._config.hidden_size // self.num_attention_heads
                    __A	:       Union[str, Any]      =							(
                        batch * self.num_attention_heads,
                        head_dim,
                        past_key_values_length,
                    )
                    __A	:       List[str]      =							(
                        batch * self.num_attention_heads,
                        past_key_values_length,
                        head_dim,
                    )
                    __A	:       Optional[Any]      =							[
                        (torch.zeros(lowerCamelCase		), torch.zeros(lowerCamelCase		)) for _ in range(self.num_layers		)
                    ]

          __A	:       int      =							common_inputs["attention_mask"]
          if self.use_past:
               __A	:       Optional[int]      =							ordered_inputs["attention_mask"].dtype
               __A	:       Any      =							torch.cat(
                   [ordered_inputs["attention_mask"], torch.ones(lowerCamelCase			,     lowerCamelCase			,     dtype=lowerCamelCase		)]			,     dim=1		)

          return ordered_inputs


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return 13





'''simple docstring'''


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCamelCase 							=  logging.get_logger(__name__)


_UpperCamelCase 							=  {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(f"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()							)}."							)

     if tokenizer_name is None:
          __A	:       Tuple      =							TOKENIZER_CLASSES
     else:
          __A	:       Dict      =							{tokenizer_name: getattr(SCREAMING_SNAKE_CASE       ,					tokenizer_name + "Fast"							)}

     logger.info(f"Loading tokenizer classes: {tokenizer_names}"							)

     for tokenizer_name in tokenizer_names:
          __A	:       int      =							TOKENIZER_CLASSES[tokenizer_name]

          __A	:       Union[str, Any]      =							True
          if checkpoint_name is None:
               __A	:       Dict      =							list(tokenizer_class.max_model_input_sizes.keys()							)
          else:
               __A	:       str      =							[checkpoint_name]

          logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"							)

          for checkpoint in checkpoint_names:
               logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}"							)

               # Load tokenizer
               __A	:       List[str]      =							tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE       ,					force_download=SCREAMING_SNAKE_CASE							)

               # Save fast tokenizer
               logger.info(f"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"							)

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    __A  ,__A	:       Optional[Any]      =							checkpoint.split("/"							)
                    __A	:       str      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
               elif add_prefix:
                    __A	:       Optional[int]      =							checkpoint
                    __A	:       Any      =							dump_path
               else:
                    __A	:       int      =							None
                    __A	:       Optional[int]      =							dump_path

               logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()							)[0]:
                    __A	:       Any      =							list(tokenizer.pretrained_vocab_files_map.values()							)[0][checkpoint]
                    __A	:       Optional[int]      =							file_path.split(SCREAMING_SNAKE_CASE							)[-1][0]
                    if next_char == "/":
                         __A	:       Union[str, Any]      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                         __A	:       Union[str, Any]      =							None

                    logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               __A	:       Tuple      =							tokenizer.save_pretrained(
                   SCREAMING_SNAKE_CASE       ,					legacy_format=SCREAMING_SNAKE_CASE       ,					filename_prefix=SCREAMING_SNAKE_CASE							)
               logger.info(f"=> File names {file_names}"							)

               for file_name in file_names:
                    if not file_name.endswith("tokenizer.json"							):
                         os.remove(SCREAMING_SNAKE_CASE							)
                         logger.info(f"=> removing {file_name}"							)


if __name__ == "__main__":
      _UpperCamelCase 							=  argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      _UpperCamelCase 							=  parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

