
import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_snake_case      = [
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_snake_case      = [
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]

def  __lowerCamelCase					( _lowercase						,     _lowercase			)       ->      List[Any]:
     UpperCamelCase						    =					{
         'word_embeddings.weight': 'word_embeddings.weight',
         'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
         'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
         'weight': 'ln_f.weight',
         'bias': 'ln_f.bias',
     }

     if key in layer_rename_map:
          return layer_rename_map[key]

     # Handle transformer blocks
     UpperCamelCase						    =					int(re.match(r'.*layer_(\d*).*'						,     _lowercase			)[1]			)
     layer_number -= 3
     return F'h.{layer_number}.' + key

def  __lowerCamelCase					( _lowercase			)       ->      Union[str, Any]:


     if dtype == torch.bool:
          return 1 / 8
     UpperCamelCase						    =					re.search(r'[^\d](\d+)$'						,     str(_lowercase			)			)
     if bit_search is None:
          raise ValueError(F'`dtype` is not a valid dtype: {dtype}.'			)
     UpperCamelCase						    =					int(bit_search.groups()[0]			)
     return bit_size // 8

def  __lowerCamelCase					( _lowercase						,     _lowercase						,     _lowercase						,     _lowercase						,     _lowercase			)       ->      Tuple:


     if bloom_config_file == "":
          UpperCamelCase						    =					BloomConfig()
     else:
          UpperCamelCase						    =					BloomConfig.from_json_file(_lowercase			)

     if shard_model:
          UpperCamelCase						    =					os.listdir(_lowercase			)
          UpperCamelCase						    =					sorted(filter(lambda _lowercase			: s.startswith('layer'			) and "model_00" in s						,     _lowercase			)			)

          UpperCamelCase						    =					{'weight_map': {}, 'metadata': {}}
          UpperCamelCase						    =					0

          UpperCamelCase						    =					None

          UpperCamelCase						    =					BloomConfig()

          for j, file in enumerate(_lowercase			):
               print('Processing file: {}'.format(_lowercase			)			)
               UpperCamelCase						    =					None

               for i in range(_lowercase			):
                    # load all TP files
                    UpperCamelCase						    =					file.replace('model_00'						,     F'model_0{i}'			)
                    UpperCamelCase						    =					torch.load(os.path.join(_lowercase						,     _lowercase			)						,     map_location='cpu'			)

                    # Rename keys in the transformers names
                    UpperCamelCase						    =					list(temp.keys()			)
                    for key in keys:
                         UpperCamelCase						    =					temp.pop(_lowercase			)

                    if tensors is None:
                         UpperCamelCase						    =					temp
                    else:
                         for key in tensors.keys():
                              if any(key.endswith(_lowercase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
                                   # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCamelCase						    =					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCamelCase						    =					torch.cat([tensors[key], temp[key]]						,     dim=_lowercase			)

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
                         UpperCamelCase						    =					tensors[key] / pretraining_tp
               torch.save(
                   _lowercase						,     os.path.join(
                       _lowercase						,     'pytorch_model_{}-of-{}.bin'.format(str(j + 1			).zfill(5			)						,     str(len(_lowercase			)			).zfill(5			)			)						,     )						,     )

               for key in tensors.keys():
                    UpperCamelCase						    =					tensors[key]
                    total_size += value.numel() * get_dtype_size(value.dtype			)
                    if key not in index_dict["weight_map"]:
                         UpperCamelCase						    =					'pytorch_model_{}-of-{}.bin'.format(
                             str(j + 1			).zfill(5			)						,     str(len(_lowercase			)			).zfill(5			)			)

          UpperCamelCase						    =					BloomConfig()
          UpperCamelCase						    =					pytorch_dump_folder_path + '/' + CONFIG_NAME
          UpperCamelCase						    =					total_size
          with open(_lowercase						,     'w'						,     encoding='utf-8'			) as f:
               f.write(config.to_json_string()			)
          with open(os.path.join(_lowercase						,     WEIGHTS_NAME + '.index.json'			)						,     'w'						,     encoding='utf-8'			) as f:
               UpperCamelCase						    =					json.dumps(_lowercase						,     indent=2						,     sort_keys=_lowercase			) + '\n'
               f.write(_lowercase			)
     else:
          UpperCamelCase						    =					BloomModel(_lowercase			)

          UpperCamelCase						    =					os.listdir(_lowercase			)
          UpperCamelCase						    =					sorted(filter(lambda _lowercase			: s.startswith('layer'			) and "model_00" in s						,     _lowercase			)			)

          UpperCamelCase						    =					None
          for i, file in enumerate(_lowercase			):
               UpperCamelCase						    =					None
               for i in range(_lowercase			):
                    # load all TP files
                    UpperCamelCase						    =					file.replace('model_00'						,     F'model_0{i}'			)
                    UpperCamelCase						    =					torch.load(os.path.join(_lowercase						,     _lowercase			)						,     map_location='cpu'			)

                    # Rename keys in the transformers names
                    UpperCamelCase						    =					list(temp.keys()			)
                    for key in keys:
                         UpperCamelCase						    =					temp.pop(_lowercase			)

                    if tensors is None:
                         UpperCamelCase						    =					temp
                    else:
                         for key in tensors.keys():
                              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                              if any(key.endswith(_lowercase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCamelCase						    =					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCamelCase						    =					torch.cat([tensors[key], temp[key]]						,     dim=_lowercase			)

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
                         UpperCamelCase						    =					tensors[key] / pretraining_tp

               UpperCamelCase						    =					model.load_state_dict(_lowercase						,     strict=_lowercase			)
               assert not other_keys.unexpected_keys, F'The keys {other_keys.unexpected_keys} are unexpected'
               if missing_keys is None:
                    UpperCamelCase						    =					set(other_keys.missing_keys			)
               else:
                    UpperCamelCase						    =					missing_keys.intersection(set(other_keys.missing_keys			)			)

          assert not missing_keys, F'The keys {missing_keys} are missing'

          # Save pytorch-model
          os.makedirs(_lowercase						,     exist_ok=_lowercase			)
          UpperCamelCase						    =					pytorch_dump_folder_path + '/' + WEIGHTS_NAME
          UpperCamelCase						    =					pytorch_dump_folder_path + '/' + CONFIG_NAME
          print(F'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'			)
          if config.torch_dtype is not None:
               UpperCamelCase						    =					model.to(config.torch_dtype			)
          torch.save(model.state_dict()						,     _lowercase			)
          print(F'Save configuration file to {pytorch_config_dump_path}'			)
          with open(_lowercase						,     'w'						,     encoding='utf-8'			) as f:
               f.write(config.to_json_string()			)


if __name__ == "__main__":
 _snake_case      = argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--bloom_checkpoint_path''',
     default=None,
     type=str,
     required=True,
     help='''Path to the Megatron-LM checkpoint path.''',
 )
 parser.add_argument(
     '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
 )
 parser.add_argument(
     '''--bloom_config_file''',
     default='''''',
     type=str,
     help=(
         '''An optional config json file corresponding to the pre-trained model. \n'''
         '''This specifies the model architecture.'''
     ),
 )
 parser.add_argument(
     '''--shard_model''',
     action='''store_true''',
     help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
 )
 parser.add_argument(
     '''--pretraining_tp''',
     default=4,
     type=int,
     help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
 )
 _snake_case      = parser.parse_args()
 convert_bloom_checkpoint_to_pytorch(
     args.bloom_checkpoint_path,
     args.bloom_config_file,
     args.pytorch_dump_folder_path,
     args.shard_model,
     args.pretraining_tp,
 )





'''simple docstring'''



import operator


def       snake_case   (	snake_case      : list					,		snake_case      : bool = False					,		snake_case      : list | None = None			) ->  list:


    """simple docstring"""


    lowerCAmelCase 						=							operator.lt if reverse else operator.gt
    lowerCAmelCase 						=							solution or []

    if not arr:
        return solution

    lowerCAmelCase 						=							[arr.pop(0			)]
    for i, item in enumerate(snake_case			):
        if _operator(snake_case					,		sublist[-1]			):
            sublist.append(snake_case			)
            arr.pop(snake_case			)

    #  merging sublist into solution list
    if not solution:
        solution.extend(snake_case			)
    else:
        while sublist:
            lowerCAmelCase 						=							sublist.pop(0			)
            for i, xx in enumerate(snake_case			):
                if not _operator(snake_case					,		snake_case			):
                    solution.insert(snake_case					,		snake_case			)
                    break
            else:
                solution.append(snake_case			)

    strand_sort(snake_case					,		snake_case					,		snake_case			)
    return solution


if __name__ == "__main__":
     assert strand_sort([4, 3, 5, 1, 2]) == [1, 2, 3, 4, 5]
     assert strand_sort([4, 3, 5, 1, 2], reverse=True) == [5, 4, 3, 2, 1]





