


'''simple docstring'''



import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

A__						:   Union[str, Any]				=    [
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

A__						:   Tuple				=    [
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Optional[Any] ,       UpperCAmelCase_       :							int				)	->						Dict:
     __lowerCamelCase  :      Optional[Any]														=						{
         'word_embeddings.weight': 'word_embeddings.weight',
         'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
         'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
         'weight': 'ln_f.weight',
         'bias': 'ln_f.bias',
     }

     if key in layer_rename_map:
          return layer_rename_map[key]

     # Handle transformer blocks
     __lowerCamelCase  :      Optional[int]														=						int(re.match(R'.*layer_(\d*).*' ,       UpperCAmelCase_				)[1]				)
     layer_number -= 3
     return F'h.{layer_number}.' + key



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Optional[Any]				)	->						int:
     if dtype == torch.bool:
          return 1 / 8
     __lowerCamelCase  :      Optional[Any]														=						re.search(R'[^\d](\d+)$' ,       str(UpperCAmelCase_				)				)
     if bit_search is None:
          raise ValueError(F'`dtype` is not a valid dtype: {dtype}.'				)
     __lowerCamelCase  :      List[str]														=						int(bit_search.groups()[0]				)
     return bit_size // 8



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Optional[Any] ,       UpperCAmelCase_       :							Optional[Any] ,       UpperCAmelCase_       :							Any ,       UpperCAmelCase_       :							Optional[Any] ,       UpperCAmelCase_       :							Optional[int]				)	->						Dict:
     # Construct model
     if bloom_config_file == "":
          __lowerCamelCase  :      Union[str, Any]														=						BloomConfig()
     else:
          __lowerCamelCase  :      List[str]														=						BloomConfig.from_json_file(UpperCAmelCase_				)

     if shard_model:
          __lowerCamelCase  :      Dict														=						os.listdir(UpperCAmelCase_				)
          __lowerCamelCase  :      Union[str, Any]														=						sorted(filter(lambda UpperCAmelCase_				: s.startswith('layer'				) and "model_00" in s ,       UpperCAmelCase_				)				)

          __lowerCamelCase  :      Any														=						{'weight_map': {}, 'metadata': {}}
          __lowerCamelCase  :      int														=						0

          __lowerCamelCase  :      int														=						None

          __lowerCamelCase  :      Dict														=						BloomConfig()

          for j, file in enumerate(UpperCAmelCase_				):
               print('Processing file: {}'.format(UpperCAmelCase_				)				)
               __lowerCamelCase  :      Optional[Any]														=						None

               for i in range(UpperCAmelCase_				):
                    # load all TP files
                    __lowerCamelCase  :      Optional[int]														=						file.replace('model_00' ,       F'model_0{i}'				)
                    __lowerCamelCase  :      Any														=						torch.load(os.path.join(UpperCAmelCase_ ,       UpperCAmelCase_				) ,       map_location='cpu'				)

                    # Rename keys in the transformers names
                    __lowerCamelCase  :      Dict														=						list(temp.keys()				)
                    for key in keys:
                         __lowerCamelCase  :      Optional[Any]														=						temp.pop(UpperCAmelCase_				)

                    if tensors is None:
                         __lowerCamelCase  :      List[str]														=						temp
                    else:
                         for key in tensors.keys():
                              if any(key.endswith(UpperCAmelCase_				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                                   # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   __lowerCamelCase  :      Tuple														=						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
                                   # We concatenate these weights accross TP ranks
                                   __lowerCamelCase  :      str														=						torch.cat([tensors[key], temp[key]] ,       dim=UpperCAmelCase_				)

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(UpperCAmelCase_				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                         __lowerCamelCase  :      List[str]														=						tensors[key] / pretraining_tp
               torch.save(
                   UpperCAmelCase_ ,       os.path.join(
                       UpperCAmelCase_ ,       'pytorch_model_{}-of-{}.bin'.format(str(j + 1				).zfill(5				) ,       str(len(UpperCAmelCase_				)				).zfill(5				)				) ,       ) ,       )

               for key in tensors.keys():
                    __lowerCamelCase  :      Tuple														=						tensors[key]
                    total_size += value.numel() * get_dtype_size(value.dtype				)
                    if key not in index_dict["weight_map"]:
                         __lowerCamelCase  :      str														=						'pytorch_model_{}-of-{}.bin'.format(
                             str(j + 1				).zfill(5				) ,       str(len(UpperCAmelCase_				)				).zfill(5				)				)

          __lowerCamelCase  :      List[Any]														=						BloomConfig()
          __lowerCamelCase  :      List[Any]														=						pytorch_dump_folder_path + '/' + CONFIG_NAME
          __lowerCamelCase  :      str														=						total_size
          with open(UpperCAmelCase_ ,       'w' ,       encoding='utf-8'				) as f:
               f.write(config.to_json_string()				)
          with open(os.path.join(UpperCAmelCase_ ,       WEIGHTS_NAME + '.index.json'				) ,       'w' ,       encoding='utf-8'				) as f:
               __lowerCamelCase  :      Tuple														=						json.dumps(UpperCAmelCase_ ,       indent=2 ,       sort_keys=UpperCAmelCase_				) + '\n'
               f.write(UpperCAmelCase_				)
     else:
          __lowerCamelCase  :      str														=						BloomModel(UpperCAmelCase_				)

          __lowerCamelCase  :      List[Any]														=						os.listdir(UpperCAmelCase_				)
          __lowerCamelCase  :      Tuple														=						sorted(filter(lambda UpperCAmelCase_				: s.startswith('layer'				) and "model_00" in s ,       UpperCAmelCase_				)				)

          __lowerCamelCase  :      List[str]														=						None
          for i, file in enumerate(UpperCAmelCase_				):
               __lowerCamelCase  :      Union[str, Any]														=						None
               for i in range(UpperCAmelCase_				):
                    # load all TP files
                    __lowerCamelCase  :      Optional[Any]														=						file.replace('model_00' ,       F'model_0{i}'				)
                    __lowerCamelCase  :      List[str]														=						torch.load(os.path.join(UpperCAmelCase_ ,       UpperCAmelCase_				) ,       map_location='cpu'				)

                    # Rename keys in the transformers names
                    __lowerCamelCase  :      List[Any]														=						list(temp.keys()				)
                    for key in keys:
                         __lowerCamelCase  :      int														=						temp.pop(UpperCAmelCase_				)

                    if tensors is None:
                         __lowerCamelCase  :      List[str]														=						temp
                    else:
                         for key in tensors.keys():
                              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                              if any(key.endswith(UpperCAmelCase_				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   __lowerCamelCase  :      Any														=						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
                                   # We concatenate these weights accross TP ranks
                                   __lowerCamelCase  :      int														=						torch.cat([tensors[key], temp[key]] ,       dim=UpperCAmelCase_				)

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(UpperCAmelCase_				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
                         __lowerCamelCase  :      Union[str, Any]														=						tensors[key] / pretraining_tp

               __lowerCamelCase  :      int														=						model.load_state_dict(UpperCAmelCase_ ,       strict=UpperCAmelCase_				)
               assert not other_keys.unexpected_keys, F'The keys {other_keys.unexpected_keys} are unexpected'
               if missing_keys is None:
                    __lowerCamelCase  :      str														=						set(other_keys.missing_keys				)
               else:
                    __lowerCamelCase  :      int														=						missing_keys.intersection(set(other_keys.missing_keys				)				)

          assert not missing_keys, F'The keys {missing_keys} are missing'

          # Save pytorch-model
          os.makedirs(UpperCAmelCase_ ,       exist_ok=UpperCAmelCase_				)
          __lowerCamelCase  :      Optional[int]														=						pytorch_dump_folder_path + '/' + WEIGHTS_NAME
          __lowerCamelCase  :      List[str]														=						pytorch_dump_folder_path + '/' + CONFIG_NAME
          print(F'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'				)
          if config.torch_dtype is not None:
               __lowerCamelCase  :      Dict														=						model.to(config.torch_dtype				)
          torch.save(model.state_dict() ,       UpperCAmelCase_				)
          print(F'Save configuration file to {pytorch_config_dump_path}'				)
          with open(UpperCAmelCase_ ,       'w' ,       encoding='utf-8'				) as f:
               f.write(config.to_json_string()				)


if __name__ == "__main__":
      A__						:   Union[str, Any]				=    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--bloom_checkpoint_path""",
          default=None,
          type=str,
          required=True,
          help="""Path to the Megatron-LM checkpoint path.""",
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
      )
      parser.add_argument(
          """--bloom_config_file""",
          default="""""",
          type=str,
          help=(
              """An optional config json file corresponding to the pre-trained model. \n"""
              """This specifies the model architecture."""
          ),
      )
      parser.add_argument(
          """--shard_model""",
          action="""store_true""",
          help="""An optional setting to shard the output model \nThis enables sharding the converted checkpoint""",
      )
      parser.add_argument(
          """--pretraining_tp""",
          default=4,
          type=int,
          help="""Pretraining TP rank that has been used when training the model in Megatron-LM \n""",
      )
      A__						:   List[str]				=    parser.parse_args()
      convert_bloom_checkpoint_to_pytorch(
          args.bloom_checkpoint_path,
          args.bloom_config_file,
          args.pytorch_dump_folder_path,
          args.shard_model,
          args.pretraining_tp,
      )










'''simple docstring'''



A__						:   dict[tuple[int, int, int], int]				=    {}



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int ,       UpperCAmelCase_       :							int ,       UpperCAmelCase_       :							int				)	->						int:

     # if we are absent twice, or late 3 consecutive days,
     # no further prize strings are possible
     if late == 3 or absent == 2:
          return 0

     # if we have no days left, and have not failed any other rules,
     # we have a prize string
     if days == 0:
          return 1

     # No easy solution, so now we need to do the recursive calculation

     # First, check if the combination is already in the cache, and
     # if yes, return the stored value from there since we already
     # know the number of possible prize strings from this point on
     __lowerCamelCase  :      List[Any]														=						(days, absent, late)
     if key in cache:
          return cache[key]

     # now we calculate the three possible ways that can unfold from
     # this point on, depending on our attendance today

     # 1) if we are late (but not absent), the "absent" counter stays as
     # it is, but the "late" counter increases by one
     __lowerCamelCase  :      Tuple														=						_calculate(days - 1 ,       UpperCAmelCase_ ,       late + 1				)

     # 2) if we are absent, the "absent" counter increases by 1, and the
     # "late" counter resets to 0
     __lowerCamelCase  :      int														=						_calculate(days - 1 ,       absent + 1 ,       0				)

     # 3) if we are on time, this resets the "late" counter and keeps the
     # absent counter
     __lowerCamelCase  :      List[Any]														=						_calculate(days - 1 ,       UpperCAmelCase_ ,       0				)

     __lowerCamelCase  :      Optional[int]														=						state_late + state_absent + state_ontime

     __lowerCamelCase  :      Union[str, Any]														=						prizestrings
     return prizestrings



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int = 30				)	->						int:
     return _calculate(UpperCAmelCase_ ,       absent=0 ,       late=0				)


if __name__ == "__main__":
      print(solution())






