





import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


_A					:				Optional[int]				=			[
    {"""dataset""": """wikipedia""", """config_name""": """20220301.de"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.en"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.fr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.frr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.it"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.simple"""},
    {"""dataset""": """snli""", """config_name""": """plain_text"""},
    {"""dataset""": """eli5""", """config_name""": """LFQA_reddit"""},
    {"""dataset""": """wiki40b""", """config_name""": """en"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.compressed"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.no_index"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.multiset.no_index"""},
    {"""dataset""": """natural_questions""", """config_name""": """default"""},
]


def 							__snake_case							(    lowerCAmelCase_=True	)				->						Union[str, Any]:
    if with_config:
        return [
            {
                "testcase_name": d["dataset"] + "/" + d["config_name"],
                "dataset": d["dataset"],
                "config_name": d["config_name"],
            }
            for d in DATASETS_ON_HF_GCP
        ]
    else:
        return [
            {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
        ]



@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__SCREAMING_SNAKE_CASE    )    )
class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    lowerCamelCase__    :		Any				=	None
    lowerCamelCase__    :		Optional[int]				=	None



    def  lowercase_					(       self   ,				A_   ,				A_     ):





        '''simple docstring'''

        with TemporaryDirectory() as tmp_dir:
            SCREAMING_SNAKE_CASE__											=			dataset_module_factory(A_   ,				cache_dir=A_     )

            SCREAMING_SNAKE_CASE__											=			import_main_class(dataset_module.module_path   ,				dataset=A_     )

            SCREAMING_SNAKE_CASE__											=			builder_cls(
                cache_dir=A_   ,				config_name=A_   ,				hash=dataset_module.hash   ,				)

            SCREAMING_SNAKE_CASE__											=			'''/'''.join(
                [
                    HF_GCP_BASE_URL,
                    builder_instance._relative_data_dir(with_hash=A_     ).replace(os.sep   ,				'''/'''     ),
                    config.DATASET_INFO_FILENAME,
                ]     )
            SCREAMING_SNAKE_CASE__											=			cached_path(A_   ,				cache_dir=A_     )
            self.assertTrue(os.path.exists(A_     )     )





@pytest.mark.integration
def 							__snake_case							(    lowerCAmelCase_	)				->						List[str]:
    SCREAMING_SNAKE_CASE__											=			tmp_path_factory.mktemp('''test_hf_gcp'''	) / '''test_wikipedia_simple'''
    SCREAMING_SNAKE_CASE__											=			dataset_module_factory('''wikipedia''' ,							cache_dir=lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			import_main_class(dataset_module.module_path	)
    SCREAMING_SNAKE_CASE__											=			builder_cls(
        cache_dir=lowerCAmelCase_ ,							config_name='''20220301.frr''' ,							hash=dataset_module.hash ,							)
    # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
    SCREAMING_SNAKE_CASE__											=			None
    builder_instance.download_and_prepare()
    SCREAMING_SNAKE_CASE__											=			builder_instance.as_dataset()
    assert ds



@pytest.mark.integration
def 							__snake_case							(    lowerCAmelCase_	)				->						Dict:
    SCREAMING_SNAKE_CASE__											=			dataset_module_factory('''wikipedia''' ,							cache_dir=lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			import_main_class(dataset_module.module_path ,							dataset=lowerCAmelCase_	)
    SCREAMING_SNAKE_CASE__											=			builder_cls(
        cache_dir=lowerCAmelCase_ ,							config_name='''20220301.frr''' ,							hash=dataset_module.hash ,							)
    SCREAMING_SNAKE_CASE__											=			builder_instance.as_streaming_dataset()
    assert ds
    assert isinstance(lowerCAmelCase_ ,							lowerCAmelCase_	)
    assert "train" in ds
    assert isinstance(ds['''train'''] ,							lowerCAmelCase_	)
    assert next(iter(ds['''train''']	)	)






import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__a							     =					[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

__a							     =					[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def lowerCamelCase__  (     _lowercase					,		_lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				List[Any]  =     {
         '''word_embeddings.weight''': '''word_embeddings.weight''',
         '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
         '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
         '''weight''': '''ln_f.weight''',
         '''bias''': '''ln_f.bias''',
     }

     if key in layer_rename_map:
          return layer_rename_map[key]

     # Handle transformer blocks
     UpperCAmelCase_     :				Union[str, Any]  =     int(re.match(r'''.*layer_(\d*).*'''					,		_lowercase )[1] )
     layer_number -= 3
     return f'''h.{layer_number}.''' + key







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if dtype == torch.bool:
          return 1 / 8
     UpperCAmelCase_     :				Any  =     re.search(r'''[^\d](\d+)$'''					,		str(_lowercase ) )
     if bit_search is None:
          raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.''' )
     UpperCAmelCase_     :				Optional[int]  =     int(bit_search.groups()[0] )
     return bit_size // 8







def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase ):

     '''simple docstring'''


     if bloom_config_file == "":
          UpperCAmelCase_     :				Tuple  =     BloomConfig()
     else:
          UpperCAmelCase_     :				Optional[int]  =     BloomConfig.from_json_file(_lowercase )

     if shard_model:
          UpperCAmelCase_     :				Any  =     os.listdir(_lowercase )
          UpperCAmelCase_     :				Union[str, Any]  =     sorted(filter(lambda _lowercase : s.startswith('''layer''' ) and "model_00" in s					,		_lowercase ) )

          UpperCAmelCase_     :				Any  =     {'''weight_map''': {}, '''metadata''': {}}
          UpperCAmelCase_     :				List[str]  =     0

          UpperCAmelCase_     :				Any  =     None

          UpperCAmelCase_     :				Optional[int]  =     BloomConfig()

          for j, file in enumerate(_lowercase ):
               print('''Processing file: {}'''.format(_lowercase ) )
               UpperCAmelCase_     :				Optional[Any]  =     None

               for i in range(_lowercase ):
                    # load all TP files
                    UpperCAmelCase_     :				Tuple  =     file.replace('''model_00'''					,		f'''model_0{i}''' )
                    UpperCAmelCase_     :				Any  =     torch.load(os.path.join(_lowercase					,		_lowercase )					,		map_location='''cpu''' )

                    # Rename keys in the transformers names
                    UpperCAmelCase_     :				Dict  =     list(temp.keys() )
                    for key in keys:
                         UpperCAmelCase_     :				Union[str, Any]  =     temp.pop(_lowercase )

                    if tensors is None:
                         UpperCAmelCase_     :				Union[str, Any]  =     temp
                    else:
                         for key in tensors.keys():
                              if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                                   # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCAmelCase_     :				int  =     1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCAmelCase_     :				Tuple  =     torch.cat([tensors[key], temp[key]]					,		dim=_lowercase )

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                         UpperCAmelCase_     :				List[str]  =     tensors[key] / pretraining_tp
               torch.save(
                   _lowercase					,		os.path.join(
                       _lowercase					,		'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1 ).zfill(5 )					,		str(len(_lowercase ) ).zfill(5 ) )					,		)					,		)

               for key in tensors.keys():
                    UpperCAmelCase_     :				Union[str, Any]  =     tensors[key]
                    total_size += value.numel() * get_dtype_size(value.dtype )
                    if key not in index_dict["weight_map"]:
                         UpperCAmelCase_     :				List[str]  =     '''pytorch_model_{}-of-{}.bin'''.format(
                             str(j + 1 ).zfill(5 )					,		str(len(_lowercase ) ).zfill(5 ) )

          UpperCAmelCase_     :				List[Any]  =     BloomConfig()
          UpperCAmelCase_     :				Tuple  =     pytorch_dump_folder_path + '''/''' + CONFIG_NAME
          UpperCAmelCase_     :				List[str]  =     total_size
          with open(_lowercase					,		'''w'''					,		encoding='''utf-8''' ) as f:
               f.write(config.to_json_string() )
          with open(os.path.join(_lowercase					,		WEIGHTS_NAME + '''.index.json''' )					,		'''w'''					,		encoding='''utf-8''' ) as f:
               UpperCAmelCase_     :				Optional[Any]  =     json.dumps(_lowercase					,		indent=2					,		sort_keys=_lowercase ) + '''\n'''
               f.write(_lowercase )
     else:
          UpperCAmelCase_     :				Any  =     BloomModel(_lowercase )

          UpperCAmelCase_     :				Tuple  =     os.listdir(_lowercase )
          UpperCAmelCase_     :				Union[str, Any]  =     sorted(filter(lambda _lowercase : s.startswith('''layer''' ) and "model_00" in s					,		_lowercase ) )

          UpperCAmelCase_     :				Any  =     None
          for i, file in enumerate(_lowercase ):
               UpperCAmelCase_     :				Optional[Any]  =     None
               for i in range(_lowercase ):
                    # load all TP files
                    UpperCAmelCase_     :				List[Any]  =     file.replace('''model_00'''					,		f'''model_0{i}''' )
                    UpperCAmelCase_     :				Optional[int]  =     torch.load(os.path.join(_lowercase					,		_lowercase )					,		map_location='''cpu''' )

                    # Rename keys in the transformers names
                    UpperCAmelCase_     :				str  =     list(temp.keys() )
                    for key in keys:
                         UpperCAmelCase_     :				Dict  =     temp.pop(_lowercase )

                    if tensors is None:
                         UpperCAmelCase_     :				int  =     temp
                    else:
                         for key in tensors.keys():
                              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                              if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCAmelCase_     :				Optional[int]  =     1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCAmelCase_     :				List[str]  =     torch.cat([tensors[key], temp[key]]					,		dim=_lowercase )

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                         UpperCAmelCase_     :				Dict  =     tensors[key] / pretraining_tp

               UpperCAmelCase_     :				Tuple  =     model.load_state_dict(_lowercase					,		strict=_lowercase )
               assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
               if missing_keys is None:
                    UpperCAmelCase_     :				Union[str, Any]  =     set(other_keys.missing_keys )
               else:
                    UpperCAmelCase_     :				Dict  =     missing_keys.intersection(set(other_keys.missing_keys ) )

          assert not missing_keys, f'''The keys {missing_keys} are missing'''

          # Save pytorch-model
          os.makedirs(_lowercase					,		exist_ok=_lowercase )
          UpperCAmelCase_     :				str  =     pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
          UpperCAmelCase_     :				Dict  =     pytorch_dump_folder_path + '''/''' + CONFIG_NAME
          print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}''' )
          if config.torch_dtype is not None:
               UpperCAmelCase_     :				Optional[int]  =     model.to(config.torch_dtype )
          torch.save(model.state_dict()					,		_lowercase )
          print(f'''Save configuration file to {pytorch_config_dump_path}''' )
          with open(_lowercase					,		'''w'''					,		encoding='''utf-8''' ) as f:
               f.write(config.to_json_string() )


if __name__ == "__main__":
     __a							     =					argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '--bloom_checkpoint_path',
         default=None,
         type=str,
         required=True,
         help='Path to the Megatron-LM checkpoint path.',
     )
     parser.add_argument(
         '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
     )
     parser.add_argument(
         '--bloom_config_file',
         default='',
         type=str,
         help=(
             'An optional config json file corresponding to the pre-trained model. \n'
             'This specifies the model architecture.'
         ),
     )
     parser.add_argument(
         '--shard_model',
         action='store_true',
         help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
     )
     parser.add_argument(
         '--pretraining_tp',
         default=4,
         type=int,
         help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
     )
     __a							     =					parser.parse_args()
     convert_bloom_checkpoint_to_pytorch(
         args.bloom_checkpoint_path,
         args.bloom_config_file,
         args.pytorch_dump_folder_path,
         args.shard_model,
         args.pretraining_tp,
     )