



'''simple docstring'''

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a							   =				get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class 						a_		(     snake_case    ,   unittest.TestCase    ):
							UpperCAmelCase			:		Tuple        =				XLNetTokenizer
							UpperCAmelCase			:		str        =				XLNetTokenizerFast
							UpperCAmelCase			:		str        =				True
							UpperCAmelCase			:		str        =				True


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					int:
										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case:    Tuple      							=XLNetTokenizer(a_      ,						keep_accents=a_			)
										tokenizer.sanitize_special_tokens()
										tokenizer.save_pretrained(self.tmpdirname			)


							def    UpperCamelCase     (				self   :						str			)			->					str:
										snake_case:    List[str]      							='<s>'
										snake_case:    Tuple      							=1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_			)      ,						a_			)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_			)      ,						a_			)


							def    UpperCamelCase     (				self   :						Any			)			->					int:
										snake_case:    str      							=list(self.get_tokenizer().get_vocab().keys()			)

										self.assertEqual(vocab_keys[0]      ,						'<unk>'			)
										self.assertEqual(vocab_keys[1]      ,						'<s>'			)
										self.assertEqual(vocab_keys[-1]      ,						'<eod>'			)
										self.assertEqual(len(a_			)      ,						1_0_0_6			)


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					List[Any]:
										self.assertEqual(self.get_tokenizer().vocab_size      ,						1_0_0_0			)


							def    UpperCamelCase     (				self   :						Union[str, Any]			)			->					Tuple:
										snake_case:    str      							=XLNetTokenizer(a_      ,						keep_accents=a_			)

										snake_case:    Dict      							=tokenizer.tokenize('This is a test'			)
										self.assertListEqual(a_      ,						['▁This', '▁is', '▁a', '▁t', 'est']			)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)      ,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]			)

										snake_case:    List[str]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        's',
										        'é',
										        '.',
										    ]      ,						)
										snake_case:    Optional[int]      							=tokenizer.convert_tokens_to_ids(a_			)
										self.assertListEqual(a_      ,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]			)

										snake_case:    List[str]      							=tokenizer.convert_ids_to_tokens(a_			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '<unk>',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        's',
										        '<unk>',
										        '.',
										    ]      ,						)


							def    UpperCamelCase     (				self   :						Dict			)			->					int:
										snake_case:    Tuple      							=XLNetTokenizer(a_      ,						do_lower_case=a_			)
										snake_case:    Optional[int]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + '',
										        'i',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        'se',
										        '.',
										    ]      ,						)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'			)      ,						['▁he', 'll', 'o']			)


							def    UpperCamelCase     (				self   :						Tuple			)			->					Dict:
										snake_case:    str      							=XLNetTokenizer(a_      ,						do_lower_case=a_			)
										snake_case:    List[Any]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        'se',
										        '.',
										    ]      ,						)


							@slow
							def    UpperCamelCase     (				self   :						List[str]			)			->					Optional[Any]:
										snake_case:    Tuple      							=XLNetTokenizer.from_pretrained('xlnet-base-cased'			)

										snake_case:    Optional[Any]      							=tokenizer.encode('sequence builders'      ,						add_special_tokens=a_			)
										snake_case:    List[str]      							=tokenizer.encode('multi-sequence build'      ,						add_special_tokens=a_			)

										snake_case:    Optional[Any]      							=tokenizer.build_inputs_with_special_tokens(a_			)
										snake_case:    Any      							=tokenizer.build_inputs_with_special_tokens(a_      ,						a_			)

										assert encoded_sentence == text + [4, 3]
										assert encoded_pair == text + [4] + text_a + [4, 3]




							@slow
							def    UpperCamelCase     (				self   :						Dict			)			->					Optional[int]:
										# fmt: off
										snake_case:    Dict      							={'input_ids': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=a_      ,						model_name='xlnet-base-cased'      ,						revision='c841166438c31ec7ca9a106dee7bb312b73ae511'      ,						)









'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a							   =				[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a							   =				[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			snake_case:    Any      							={
			    'word_embeddings.weight': 'word_embeddings.weight',
			    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
			    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
			    'weight': 'ln_f.weight',
			    'bias': 'ln_f.bias',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			snake_case:    Dict      							=int(re.match(R'.*layer_(\d*).*'				,							__UpperCAmelCase					)[1]					)
			layer_number -= 3
			return f'''h.{layer_number}.''' + key







def a_     (    __UpperCAmelCase					)				->					Union[str, Any]:





			"""simple docstring"""







			if dtype == torch.bool:
						return 1 / 8
			snake_case:    Optional[Any]      							=re.search(R'[^\d](\d+)$'				,							str(__UpperCAmelCase					)					)
			if bit_search is None:
						raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''					)
			snake_case:    str      							=int(bit_search.groups()[0]					)
			return bit_size // 8







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			if bloom_config_file == "":
						snake_case:    Tuple      							=BloomConfig()
			else:
						snake_case:    List[Any]      							=BloomConfig.from_json_file(__UpperCAmelCase					)

			if shard_model:
						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Any      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    int      							={'weight_map': {}, 'metadata': {}}
						snake_case:    Dict      							=0

						snake_case:    List[str]      							=None

						snake_case:    Any      							=BloomConfig()

						for j, file in enumerate(__UpperCAmelCase					):
									print('Processing file: {}'.format(__UpperCAmelCase					)					)
									snake_case:    Any      							=None

									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    Optional[int]      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    List[str]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    int      							=list(temp.keys()					)
												for key in keys:
															snake_case:    List[Any]      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Tuple      							=temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    Optional[int]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    str      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    Dict      							=tensors[key] / pretraining_tp
									torch.save(
									    __UpperCAmelCase				,							os.path.join(
									        __UpperCAmelCase				,							'pytorch_model_{}-of-{}.bin'.format(str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)				,							)				,							)

									for key in tensors.keys():
												snake_case:    List[str]      							=tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype					)
												if key not in index_dict["weight_map"]:
															snake_case:    List[Any]      							='pytorch_model_{}-of-{}.bin'.format(
															    str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)

						snake_case:    Tuple      							=BloomConfig()
						snake_case:    Union[str, Any]      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						snake_case:    Any      							=total_size
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)
						with open(os.path.join(__UpperCAmelCase				,							WEIGHTS_NAME + '.index.json'					)				,							'w'				,							encoding='utf-8'					) as f:
									snake_case:    Tuple      							=json.dumps(__UpperCAmelCase				,							indent=2				,							sort_keys=__UpperCAmelCase					) + '\n'
									f.write(__UpperCAmelCase					)
			else:
						snake_case:    Optional[Any]      							=BloomModel(__UpperCAmelCase					)

						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Optional[int]      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    str      							=None
						for i, file in enumerate(__UpperCAmelCase					):
									snake_case:    List[str]      							=None
									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    int      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    Optional[Any]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    List[str]      							=list(temp.keys()					)
												for key in keys:
															snake_case:    int      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Any      							=temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    List[Any]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    Tuple      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    int      							=tensors[key] / pretraining_tp

									snake_case:    Optional[Any]      							=model.load_state_dict(__UpperCAmelCase				,							strict=__UpperCAmelCase					)
									assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												snake_case:    Optional[Any]      							=set(other_keys.missing_keys					)
									else:
												snake_case:    Any      							=missing_keys.intersection(set(other_keys.missing_keys					)					)

						assert not missing_keys, f'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(__UpperCAmelCase				,							exist_ok=__UpperCAmelCase					)
						snake_case:    Optional[int]      							=pytorch_dump_folder_path + '/' + WEIGHTS_NAME
						snake_case:    Any      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''					)
						if config.torch_dtype is not None:
									snake_case:    Optional[Any]      							=model.to(config.torch_dtype					)
						torch.save(model.state_dict()				,							__UpperCAmelCase					)
						print(f'''Save configuration file to {pytorch_config_dump_path}'''					)
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)


if __name__ == "__main__":
	a							   =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--bloom_checkpoint_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the Megatron-LM checkpoint path.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--bloom_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--shard_model',
	    action='store_true',
	    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
	)
	parser.add_argument(
	    '--pretraining_tp',
	    default=4,
	    type=int,
	    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
	)
	a							   =				parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)




