





'''simple docstring'''






import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

A_		:  Optional[int]		 =						[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

A_		:  Optional[int]		 =						[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]



def        UpperCamelCase__		(						__magic_name__  :   str		,       __magic_name__  :   Optional[Any]      )				->			Tuple:
 '''simple docstring'''






 snake_case__     :					Optional[Any]    =  {
     """word_embeddings.weight""": """word_embeddings.weight""",
     """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
     """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
     """weight""": """ln_f.weight""",
     """bias""": """ln_f.bias""",
 }

 if key in layer_rename_map:
  return layer_rename_map[key]

 # Handle transformer blocks
 snake_case__     :					int    =  int(re.match(R""".*layer_(\d*).*"""		,       __magic_name__      )[1]      )
 layer_number -= 3
 return f"h.{layer_number}." + key



def        UpperCamelCase__		(						__magic_name__  :   List[str]      )				->			Tuple:
 '''simple docstring'''






 if dtype == torch.bool:
  return 1 / 8
 snake_case__     :					List[Any]    =  re.search(R"""[^\d](\d+)$"""		,       str(__magic_name__      )      )
 if bit_search is None:
  raise ValueError(f"`dtype` is not a valid dtype: {dtype}."      )
 snake_case__     :					Optional[Any]    =  int(bit_search.groups()[0]      )
 return bit_size // 8



def        UpperCamelCase__		(						__magic_name__  :   Dict		,       __magic_name__  :   Dict		,       __magic_name__  :   List[Any]		,       __magic_name__  :   Optional[int]		,       __magic_name__  :   Optional[int]      )				->			Optional[Any]:
 '''simple docstring'''






 if bloom_config_file == "":
  snake_case__     :					Tuple    =  BloomConfig()
 else:
  snake_case__     :					str    =  BloomConfig.from_json_file(__magic_name__      )

 if shard_model:
  snake_case__     :					str    =  os.listdir(__magic_name__      )
  snake_case__     :					Union[str, Any]    =  sorted(filter(lambda __magic_name__      : s.startswith("""layer"""      ) and "model_00" in s		,       __magic_name__      )      )

  snake_case__     :					List[Any]    =  {"""weight_map""": {}, """metadata""": {}}
  snake_case__     :					Any    =  0

  snake_case__     :					Optional[int]    =  None

  snake_case__     :					Tuple    =  BloomConfig()

  for j, file in enumerate(__magic_name__      ):
   print("""Processing file: {}""".format(__magic_name__      )      )
   snake_case__     :					Any    =  None

   for i in range(__magic_name__      ):
    # load all TP files
    snake_case__     :					int    =  file.replace("""model_00"""		,       f"model_0{i}"      )
    snake_case__     :					Union[str, Any]    =  torch.load(os.path.join(__magic_name__		,       __magic_name__      )		,       map_location="""cpu"""      )

    # Rename keys in the transformers names
    snake_case__     :					Tuple    =  list(temp.keys()      )
    for key in keys:
     snake_case__     :					str    =  temp.pop(__magic_name__      )

    if tensors is None:
     snake_case__     :					List[Any]    =  temp
    else:
     for key in tensors.keys():
      if any(key.endswith(__magic_name__      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
       # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       snake_case__     :					str    =  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
       # We concatenate these weights accross TP ranks
       snake_case__     :					str    =  torch.cat([tensors[key], temp[key]]		,       dim=__magic_name__      )

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(__magic_name__      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
     snake_case__     :					Tuple    =  tensors[key] / pretraining_tp
   torch.save(
       __magic_name__		,       os.path.join(
           __magic_name__		,       """pytorch_model_{}-of-{}.bin""".format(str(j + 1      ).zfill(5      )		,       str(len(__magic_name__      )      ).zfill(5      )      )		,       )		,       )

   for key in tensors.keys():
    snake_case__     :					int    =  tensors[key]
    total_size += value.numel() * get_dtype_size(value.dtype      )
    if key not in index_dict["weight_map"]:
     snake_case__     :					Union[str, Any]    =  """pytorch_model_{}-of-{}.bin""".format(
         str(j + 1      ).zfill(5      )		,       str(len(__magic_name__      )      ).zfill(5      )      )

  snake_case__     :					Dict    =  BloomConfig()
  snake_case__     :					Dict    =  pytorch_dump_folder_path + """/""" + CONFIG_NAME
  snake_case__     :					int    =  total_size
  with open(__magic_name__		,       """w"""		,       encoding="""utf-8"""      ) as f:
   f.write(config.to_json_string()      )
  with open(os.path.join(__magic_name__		,       WEIGHTS_NAME + """.index.json"""      )		,       """w"""		,       encoding="""utf-8"""      ) as f:
   snake_case__     :					Tuple    =  json.dumps(__magic_name__		,       indent=2		,       sort_keys=__magic_name__      ) + """\n"""
   f.write(__magic_name__      )
 else:
  snake_case__     :					Optional[Any]    =  BloomModel(__magic_name__      )

  snake_case__     :					Optional[Any]    =  os.listdir(__magic_name__      )
  snake_case__     :					Union[str, Any]    =  sorted(filter(lambda __magic_name__      : s.startswith("""layer"""      ) and "model_00" in s		,       __magic_name__      )      )

  snake_case__     :					Tuple    =  None
  for i, file in enumerate(__magic_name__      ):
   snake_case__     :					Dict    =  None
   for i in range(__magic_name__      ):
    # load all TP files
    snake_case__     :					List[str]    =  file.replace("""model_00"""		,       f"model_0{i}"      )
    snake_case__     :					Optional[int]    =  torch.load(os.path.join(__magic_name__		,       __magic_name__      )		,       map_location="""cpu"""      )

    # Rename keys in the transformers names
    snake_case__     :					Optional[int]    =  list(temp.keys()      )
    for key in keys:
     snake_case__     :					int    =  temp.pop(__magic_name__      )

    if tensors is None:
     snake_case__     :					str    =  temp
    else:
     for key in tensors.keys():
      # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
      if any(key.endswith(__magic_name__      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
       tensors[key] += temp[key]
      else:
       # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
       snake_case__     :					Optional[int]    =  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
       # We concatenate these weights accross TP ranks
       snake_case__     :					Tuple    =  torch.cat([tensors[key], temp[key]]		,       dim=__magic_name__      )

            # Divide by the number of TP the weights we want to average
   for key in tensors.keys():
    if any(key.endswith(__magic_name__      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
     snake_case__     :					str    =  tensors[key] / pretraining_tp

   snake_case__     :					Any    =  model.load_state_dict(__magic_name__		,       strict=__magic_name__      )
   assert not other_keys.unexpected_keys, f"The keys {other_keys.unexpected_keys} are unexpected"
   if missing_keys is None:
    snake_case__     :					Optional[int]    =  set(other_keys.missing_keys      )
   else:
    snake_case__     :					Optional[int]    =  missing_keys.intersection(set(other_keys.missing_keys      )      )

  assert not missing_keys, f"The keys {missing_keys} are missing"

  # Save pytorch-model
  os.makedirs(__magic_name__		,       exist_ok=__magic_name__      )
  snake_case__     :					Tuple    =  pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
  snake_case__     :					List[Any]    =  pytorch_dump_folder_path + """/""" + CONFIG_NAME
  print(f"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"      )
  if config.torch_dtype is not None:
   snake_case__     :					int    =  model.to(config.torch_dtype      )
  torch.save(model.state_dict()		,       __magic_name__      )
  print(f"Save configuration file to {pytorch_config_dump_path}"      )
  with open(__magic_name__		,       """w"""		,       encoding="""utf-8"""      ) as f:
   f.write(config.to_json_string()      )


if __name__ == "__main__":
    A_		:  Dict		 =						argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--bloom_checkpoint_path",
        default=None,
        type=str,
        required=True,
        help="Path to the Megatron-LM checkpoint path.",
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    parser.add_argument(
        "--bloom_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--shard_model",
        action="store_true",
        help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
    )
    parser.add_argument(
        "--pretraining_tp",
        default=4,
        type=int,
        help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
    )
    A_		:  Optional[int]		 =						parser.parse_args()
    convert_bloom_checkpoint_to_pytorch(
        args.bloom_checkpoint_path,
        args.bloom_config_file,
        args.pytorch_dump_folder_path,
        args.shard_model,
        args.pretraining_tp,
    )












from typing import Dict

from .base import GenericTensor, Pipeline
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				def 							lowercase__  (	self		:       Any,    lowerCamelCase		:       Tuple=None,    lowerCamelCase		:       Union[str, Any]=None,    lowerCamelCase		:       Union[str, Any]=None,    **lowerCamelCase		:       List[Any]    ):

								'''simple docstring'''
								if tokenize_kwargs is None:
												lowercase__  					=     {}

								if truncation is not None:
												if "truncation" in tokenize_kwargs:
																raise ValueError(
																    '''truncation parameter defined twice (given as keyword argument as well as in tokenize_kwargs)'''    )
												lowercase__  					=     truncation

								lowercase__  					=     tokenize_kwargs

								lowercase__  					=     {}
								if return_tensors is not None:
												lowercase__  					=     return_tensors

								return preprocess_params, {}, postprocess_params




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       Optional[int],    **lowerCamelCase		:       List[Any]    ):

								'''simple docstring'''
								lowercase__  					=     self.framework
								lowercase__  					=     self.tokenizer(lowerCamelCase,    return_tensors=lowerCamelCase,    **lowerCamelCase    )
								return model_inputs




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     self.model(**lowerCamelCase    )
								return model_outputs




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       Dict,    lowerCamelCase		:       Tuple=False    ):

								'''simple docstring'''
								# [0] is the first available tensor, logits or last_hidden_state.
								if return_tensors:
												return model_outputs[0]
								if self.framework == "pt":
												return model_outputs[0].tolist()
								elif self.framework == "tf":
												return model_outputs[0].numpy().tolist()




				def __call__(	self		:       List[str],    *lowerCamelCase		:       List[Any],    **lowerCamelCase		:       str    ):

								'''simple docstring'''
								return super().__call__(*lowerCamelCase,    **lowerCamelCase    )




