




import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFEfficientFormerForImageClassification,
         TFEfficientFormerForImageClassificationWithTeacher,
         TFEfficientFormerModel,
     )
     from transformers.models.efficientformer.modeling_tf_efficientformer import (
         TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )


if is_vision_available():
     from PIL import Image

     from transformers import EfficientFormerImageProcessor

class     __a:






    """simple docstring"""





    def __init__( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE = 13  ,_SCREAMING_SNAKE_CASE = 64  ,_SCREAMING_SNAKE_CASE = 2  ,_SCREAMING_SNAKE_CASE = 3  ,_SCREAMING_SNAKE_CASE = 3  ,_SCREAMING_SNAKE_CASE = True  ,_SCREAMING_SNAKE_CASE = True  ,_SCREAMING_SNAKE_CASE = 128  ,_SCREAMING_SNAKE_CASE=[16, 32, 64, 128]  ,_SCREAMING_SNAKE_CASE = 7  ,_SCREAMING_SNAKE_CASE = 4  ,_SCREAMING_SNAKE_CASE = 37  ,_SCREAMING_SNAKE_CASE = "gelu"  ,_SCREAMING_SNAKE_CASE = 0.1  ,_SCREAMING_SNAKE_CASE = 0.1  ,_SCREAMING_SNAKE_CASE = 10  ,_SCREAMING_SNAKE_CASE = 0.02  ,_SCREAMING_SNAKE_CASE = 2  ,_SCREAMING_SNAKE_CASE = 1  ,_SCREAMING_SNAKE_CASE = 128  ,_SCREAMING_SNAKE_CASE = [2, 2, 2, 2]  ,_SCREAMING_SNAKE_CASE = 2  ,_SCREAMING_SNAKE_CASE = 2  ,)					->						Tuple:
         UpperCAmelCase_     :				Union[str, Any]  =     parent
         UpperCAmelCase_     :				str  =     batch_size
         UpperCAmelCase_     :				Optional[int]  =     image_size
         UpperCAmelCase_     :				List[Any]  =     patch_size
         UpperCAmelCase_     :				Optional[Any]  =     num_channels
         UpperCAmelCase_     :				List[str]  =     is_training
         UpperCAmelCase_     :				Tuple  =     use_labels
         UpperCAmelCase_     :				Any  =     hidden_size
         UpperCAmelCase_     :				List[str]  =     num_hidden_layers
         UpperCAmelCase_     :				Optional[Any]  =     num_attention_heads
         UpperCAmelCase_     :				str  =     intermediate_size
         UpperCAmelCase_     :				Dict  =     hidden_act
         UpperCAmelCase_     :				Any  =     hidden_dropout_prob
         UpperCAmelCase_     :				List[Any]  =     attention_probs_dropout_prob
         UpperCAmelCase_     :				str  =     type_sequence_label_size
         UpperCAmelCase_     :				int  =     initializer_range
         UpperCAmelCase_     :				Optional[Any]  =     encoder_stride
         UpperCAmelCase_     :				Optional[int]  =     num_attention_outputs
         UpperCAmelCase_     :				Optional[Any]  =     embed_dim
         UpperCAmelCase_     :				Union[str, Any]  =     embed_dim + 1
         UpperCAmelCase_     :				Union[str, Any]  =     resolution
         UpperCAmelCase_     :				List[str]  =     depths
         UpperCAmelCase_     :				Optional[int]  =     hidden_sizes
         UpperCAmelCase_     :				int  =     dim
         UpperCAmelCase_     :				Optional[int]  =     mlp_expansion_ratio





    def 				a__ ( self    )					->						str:
         UpperCAmelCase_     :				List[str]  =     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

         UpperCAmelCase_     :				List[Any]  =     None
         if self.use_labels:
              UpperCAmelCase_     :				str  =     ids_tensor([self.batch_size]  ,self.type_sequence_label_size    )

         UpperCAmelCase_     :				List[Any]  =     self.get_config()

         return config, pixel_values, labels





    def 				a__ ( self    )					->						Any:
         return EfficientFormerConfig(
             image_size=self.image_size  ,patch_size=self.patch_size  ,num_channels=self.num_channels  ,hidden_size=self.hidden_size  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,is_decoder=_SCREAMING_SNAKE_CASE  ,initializer_range=self.initializer_range  ,encoder_stride=self.encoder_stride  ,resolution=self.resolution  ,depths=self.depths  ,hidden_sizes=self.hidden_sizes  ,dim=self.dim  ,mlp_expansion_ratio=self.mlp_expansion_ratio  ,)





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )					->						str:
         UpperCAmelCase_     :				Optional[int]  =     TFEfficientFormerModel(config=_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Any  =     model(_SCREAMING_SNAKE_CASE  ,training=_SCREAMING_SNAKE_CASE    )
         self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )					->						Union[str, Any]:
         UpperCAmelCase_     :				Tuple  =     self.type_sequence_label_size
         UpperCAmelCase_     :				Dict  =     TFEfficientFormerForImageClassification(_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Dict  =     model(_SCREAMING_SNAKE_CASE  ,labels=_SCREAMING_SNAKE_CASE  ,training=_SCREAMING_SNAKE_CASE    )
         self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.type_sequence_label_size)    )

         # test greyscale images
         UpperCAmelCase_     :				str  =     1
         UpperCAmelCase_     :				str  =     TFEfficientFormerForImageClassification(_SCREAMING_SNAKE_CASE    )

         UpperCAmelCase_     :				Any  =     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]    )
         UpperCAmelCase_     :				Dict  =     model(_SCREAMING_SNAKE_CASE  ,labels=_SCREAMING_SNAKE_CASE    )
         self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.type_sequence_label_size)    )





    def 				a__ ( self    )					->						str:
         UpperCAmelCase_     :				int  =     self.prepare_config_and_inputs()
         UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_     :				Any  =     config_and_inputs
         UpperCAmelCase_     :				Dict  =     {'''pixel_values''': pixel_values}
         return config, inputs_dict


@require_tf
class     __a(	_a							,		_a							,		unittest.TestCase  ):






    """simple docstring"""





    lowerCAmelCase         = (
        (
            TFEfficientFormerModel,
            TFEfficientFormerForImageClassificationWithTeacher,
            TFEfficientFormerForImageClassification,
        )
        if is_tf_available()
        else ()
    )
    lowerCAmelCase         = (
        {
            '''feature-extraction''': TFEfficientFormerModel,
            '''image-classification''': (
                TFEfficientFormerForImageClassification,
                TFEfficientFormerForImageClassificationWithTeacher,
            ),
        }
        if is_tf_available()
        else {}
    )

    lowerCAmelCase         = False

    lowerCAmelCase         = False
    lowerCAmelCase         = False
    lowerCAmelCase         = False
    lowerCAmelCase         = False





    def 				a__ ( self    )					->						Union[str, Any]:
         UpperCAmelCase_     :				Optional[Any]  =     TFEfficientFormerModelTester(self    )
         UpperCAmelCase_     :				Union[str, Any]  =     ConfigTester(
             self  ,config_class=_SCREAMING_SNAKE_CASE  ,has_text_modality=_SCREAMING_SNAKE_CASE  ,hidden_size=37    )





    def 				a__ ( self    )					->						str:
         self.config_tester.run_common_tests()





    @unittest.skip(reason='''EfficientFormer does not use inputs_embeds'''    )
    def 				a__ ( self    )					->						str:
         pass





    @unittest.skip(reason='''EfficientFormer does not support input and output embeddings'''    )
    def 				a__ ( self    )					->						Union[str, Any]:
         pass





    def 				a__ ( self    )					->						Any:
         UpperCAmelCase_,       UpperCAmelCase_     :				List[Any]  =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              UpperCAmelCase_     :				Any  =     model_class(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				List[Any]  =     inspect.signature(model.call    )
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              UpperCAmelCase_     :				List[str]  =     [*signature.parameters.keys()]

              UpperCAmelCase_     :				Any  =     ['''pixel_values''']
              self.assertListEqual(arg_names[:1]  ,_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self    )					->						List[Any]:
         def check_hidden_states_output(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
              UpperCAmelCase_     :				str  =     model_class(_SCREAMING_SNAKE_CASE    )

              UpperCAmelCase_     :				Dict  =     model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )  ,training=_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				Union[str, Any]  =     outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

              UpperCAmelCase_     :				List[str]  =     getattr(
                  self.model_tester  ,'''expected_num_hidden_layers'''  ,self.model_tester.num_hidden_layers + 1    )
              self.assertEqual(len(_SCREAMING_SNAKE_CASE    )  ,_SCREAMING_SNAKE_CASE    )

              if hasattr(self.model_tester  ,'''encoder_seq_length'''    ):
                   UpperCAmelCase_     :				int  =     self.model_tester.encoder_seq_length
                   if hasattr(self.model_tester  ,'''chunk_length'''    ) and self.model_tester.chunk_length > 1:
                        UpperCAmelCase_     :				Optional[Any]  =     seq_length * self.model_tester.chunk_length
              else:
                   UpperCAmelCase_     :				Union[str, Any]  =     self.model_tester.seq_length

              self.assertListEqual(
                  list(hidden_states[-1].shape[-2:]    )  ,[seq_length, self.model_tester.hidden_size]  ,)

              if config.is_encoder_decoder:
                   UpperCAmelCase_     :				Tuple  =     outputs.decoder_hidden_states

                   self.asseretIsInstance(_SCREAMING_SNAKE_CASE  ,(list, tuple)    )
                   self.assertEqual(len(_SCREAMING_SNAKE_CASE    )  ,_SCREAMING_SNAKE_CASE    )
                   UpperCAmelCase_     :				Any  =     getattr(self.model_tester  ,'''seq_length'''  ,_SCREAMING_SNAKE_CASE    )
                   UpperCAmelCase_     :				str  =     getattr(self.model_tester  ,'''decoder_seq_length'''  ,_SCREAMING_SNAKE_CASE    )

                   self.assertListEqual(
                       list(hidden_states[-1].shape[-2:]    )  ,[decoder_seq_length, self.model_tester.hidden_size]  ,)

         UpperCAmelCase_,       UpperCAmelCase_     :				Union[str, Any]  =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              UpperCAmelCase_     :				int  =     True
              check_hidden_states_output(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

              # check that output_hidden_states also work using config
              del inputs_dict["output_hidden_states"]
              UpperCAmelCase_     :				Tuple  =     True

              check_hidden_states_output(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE=False    )					->						Any:
         UpperCAmelCase_     :				Optional[int]  =     super()._prepare_for_class(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,return_labels=_SCREAMING_SNAKE_CASE    )

         if return_labels:
              if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
                   del inputs_dict["labels"]

         return inputs_dict





    def 				a__ ( self    )					->						Dict:
         UpperCAmelCase_     :				Tuple  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE    )





    @unittest.skip(reason='''EfficientFormer does not implement masked image modeling yet'''    )
    def 				a__ ( self    )					->						Optional[int]:
         UpperCAmelCase_     :				Dict  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_image_modeling(*_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self    )					->						Optional[int]:
         UpperCAmelCase_     :				Union[str, Any]  =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*_SCREAMING_SNAKE_CASE    )





    @slow
    def 				a__ ( self    )					->						str:
         for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              UpperCAmelCase_     :				Union[str, Any]  =     TFEfficientFormerModel.from_pretrained(_SCREAMING_SNAKE_CASE    )
              self.assertIsNotNone(_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self    )					->						Any:
         UpperCAmelCase_,       UpperCAmelCase_     :				Optional[int]  =     self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase_     :				List[str]  =     True

         UpperCAmelCase_     :				List[Any]  =     getattr(self.model_tester  ,'''seq_length'''  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				List[Any]  =     getattr(self.model_tester  ,'''encoder_seq_length'''  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Tuple  =     getattr(self.model_tester  ,'''key_length'''  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				str  =     getattr(self.model_tester  ,'''chunk_length'''  ,_SCREAMING_SNAKE_CASE    )

         if chunk_length is not None and hasattr(self.model_tester  ,'''num_hashes'''    ):
              UpperCAmelCase_     :				List[str]  =     encoder_seq_length * self.model_tester.num_hashes

         for model_class in self.all_model_classes:
              UpperCAmelCase_     :				Optional[int]  =     True
              UpperCAmelCase_     :				Dict  =     False
              UpperCAmelCase_     :				Any  =     True
              UpperCAmelCase_     :				Optional[Any]  =     model_class(_SCREAMING_SNAKE_CASE    )

              UpperCAmelCase_     :				Any  =     model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )  ,training=_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				Optional[int]  =     outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
              self.assertEqual(len(_SCREAMING_SNAKE_CASE    )  ,self.model_tester.num_attention_outputs    )

              # check that output_attentions also work using config
              del inputs_dict["output_attentions"]
              UpperCAmelCase_     :				Dict  =     True
              UpperCAmelCase_     :				int  =     model_class(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				List[Any]  =     model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )  ,training=_SCREAMING_SNAKE_CASE    )

              UpperCAmelCase_     :				Union[str, Any]  =     outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
              self.assertEqual(len(_SCREAMING_SNAKE_CASE    )  ,self.model_tester.num_attention_outputs    )

              if chunk_length is not None:
                   self.assertListEqual(
                       list(attentions[0].shape[-4:]    )  ,[self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]  ,)
              else:
                   self.assertListEqual(
                       list(attentions[0].shape[-3:]    )  ,[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]  ,)





    def 				a__ ( self    )					->						List[str]:
         # We use a simplified version of this test for EfficientFormer because it requires training=False
         # and Keras refuses to let us force that during functional construction
         UpperCAmelCase_,       UpperCAmelCase_     :				Any  =     self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              # Prepare our model
              UpperCAmelCase_     :				str  =     model_class(_SCREAMING_SNAKE_CASE    )
              # These are maximally general inputs for the model, with multiple None dimensions
              # Hopefully this will catch any conditionals that fail for flexible shapes
              UpperCAmelCase_     :				Any  =     {
                  key: tf.keras.Input(shape=val.shape[1:]  ,dtype=val.dtype  ,name=_SCREAMING_SNAKE_CASE    )
                  for key, val in model.input_signature.items()
                  if key in model.dummy_inputs
              }
              UpperCAmelCase_     :				List[Any]  =     model(_SCREAMING_SNAKE_CASE    )
              self.assertTrue(outputs_dict is not None    )







def lowerCamelCase__  (     ):

     '''simple docstring'''


     UpperCAmelCase_     :				Dict  =     Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''' )
     return image



@require_tf
@require_vision
class     __a(	unittest.TestCase  ):






    """simple docstring"""





    @cached_property
    def 				a__ ( self    )					->						Tuple:
         return (
             EfficientFormerImageProcessor.from_pretrained('''snap-research/efficientformer-l1-300'''    )
             if is_vision_available()
             else None
         )





    @slow
    def 				a__ ( self    )					->						int:
         UpperCAmelCase_     :				int  =     TFEfficientFormerForImageClassification.from_pretrained('''snap-research/efficientformer-l1-300'''    )
         UpperCAmelCase_     :				List[Any]  =     self.default_image_processor
         UpperCAmelCase_     :				List[str]  =     prepare_img()
         UpperCAmelCase_     :				str  =     image_processor(images=_SCREAMING_SNAKE_CASE  ,return_tensors='''tf'''    )
         # forward pass
         UpperCAmelCase_     :				Optional[int]  =     model(**_SCREAMING_SNAKE_CASE  ,training=_SCREAMING_SNAKE_CASE    )
         # verify the logits
         UpperCAmelCase_     :				str  =     tf.TensorShape((1, 1_000)    )
         self.assertEqual(outputs.logits.shape  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Tuple  =     tf.constant([-0.05_55, 0.48_25, -0.08_52]    )
         self.assertTrue(np.allclose(outputs.logits[0, :3]  ,_SCREAMING_SNAKE_CASE  ,atol=1e-4    )    )





    @slow
    def 				a__ ( self    )					->						str:
         UpperCAmelCase_     :				int  =     TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
             '''snap-research/efficientformer-l1-300'''    )
         UpperCAmelCase_     :				str  =     self.default_image_processor
         UpperCAmelCase_     :				Optional[Any]  =     prepare_img()
         UpperCAmelCase_     :				Union[str, Any]  =     image_processor(images=_SCREAMING_SNAKE_CASE  ,return_tensors='''tf'''    )
         # forward pass
         UpperCAmelCase_     :				Any  =     model(**_SCREAMING_SNAKE_CASE  ,training=_SCREAMING_SNAKE_CASE    )
         # verify the logits
         UpperCAmelCase_     :				Optional[Any]  =     tf.TensorShape((1, 1_000)    )
         self.assertEqual(outputs.logits.shape  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Tuple  =     tf.constant([-0.13_12, 0.43_53, -1.04_99]    )
         self.assertTrue(np.allclose(outputs.logits[0, :3]  ,_SCREAMING_SNAKE_CASE  ,atol=1e-4    )    )





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__a							     =					[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

__a							     =					[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def lowerCamelCase__  (     _lowercase					,		_lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				List[Any]  =     {
         '''word_embeddings.weight''': '''word_embeddings.weight''',
         '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
         '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
         '''weight''': '''ln_f.weight''',
         '''bias''': '''ln_f.bias''',
     }

     if key in layer_rename_map:
          return layer_rename_map[key]

     # Handle transformer blocks
     UpperCAmelCase_     :				Union[str, Any]  =     int(re.match(r'''.*layer_(\d*).*'''					,		_lowercase )[1] )
     layer_number -= 3
     return f'''h.{layer_number}.''' + key







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     if dtype == torch.bool:
          return 1 / 8
     UpperCAmelCase_     :				Any  =     re.search(r'''[^\d](\d+)$'''					,		str(_lowercase ) )
     if bit_search is None:
          raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.''' )
     UpperCAmelCase_     :				Optional[int]  =     int(bit_search.groups()[0] )
     return bit_size // 8







def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase ):

     '''simple docstring'''


     if bloom_config_file == "":
          UpperCAmelCase_     :				Tuple  =     BloomConfig()
     else:
          UpperCAmelCase_     :				Optional[int]  =     BloomConfig.from_json_file(_lowercase )

     if shard_model:
          UpperCAmelCase_     :				Any  =     os.listdir(_lowercase )
          UpperCAmelCase_     :				Union[str, Any]  =     sorted(filter(lambda _lowercase : s.startswith('''layer''' ) and "model_00" in s					,		_lowercase ) )

          UpperCAmelCase_     :				Any  =     {'''weight_map''': {}, '''metadata''': {}}
          UpperCAmelCase_     :				List[str]  =     0

          UpperCAmelCase_     :				Any  =     None

          UpperCAmelCase_     :				Optional[int]  =     BloomConfig()

          for j, file in enumerate(_lowercase ):
               print('''Processing file: {}'''.format(_lowercase ) )
               UpperCAmelCase_     :				Optional[Any]  =     None

               for i in range(_lowercase ):
                    # load all TP files
                    UpperCAmelCase_     :				Tuple  =     file.replace('''model_00'''					,		f'''model_0{i}''' )
                    UpperCAmelCase_     :				Any  =     torch.load(os.path.join(_lowercase					,		_lowercase )					,		map_location='''cpu''' )

                    # Rename keys in the transformers names
                    UpperCAmelCase_     :				Dict  =     list(temp.keys() )
                    for key in keys:
                         UpperCAmelCase_     :				Union[str, Any]  =     temp.pop(_lowercase )

                    if tensors is None:
                         UpperCAmelCase_     :				Union[str, Any]  =     temp
                    else:
                         for key in tensors.keys():
                              if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                                   # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCAmelCase_     :				int  =     1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCAmelCase_     :				Tuple  =     torch.cat([tensors[key], temp[key]]					,		dim=_lowercase )

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                         UpperCAmelCase_     :				List[str]  =     tensors[key] / pretraining_tp
               torch.save(
                   _lowercase					,		os.path.join(
                       _lowercase					,		'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1 ).zfill(5 )					,		str(len(_lowercase ) ).zfill(5 ) )					,		)					,		)

               for key in tensors.keys():
                    UpperCAmelCase_     :				Union[str, Any]  =     tensors[key]
                    total_size += value.numel() * get_dtype_size(value.dtype )
                    if key not in index_dict["weight_map"]:
                         UpperCAmelCase_     :				List[str]  =     '''pytorch_model_{}-of-{}.bin'''.format(
                             str(j + 1 ).zfill(5 )					,		str(len(_lowercase ) ).zfill(5 ) )

          UpperCAmelCase_     :				List[Any]  =     BloomConfig()
          UpperCAmelCase_     :				Tuple  =     pytorch_dump_folder_path + '''/''' + CONFIG_NAME
          UpperCAmelCase_     :				List[str]  =     total_size
          with open(_lowercase					,		'''w'''					,		encoding='''utf-8''' ) as f:
               f.write(config.to_json_string() )
          with open(os.path.join(_lowercase					,		WEIGHTS_NAME + '''.index.json''' )					,		'''w'''					,		encoding='''utf-8''' ) as f:
               UpperCAmelCase_     :				Optional[Any]  =     json.dumps(_lowercase					,		indent=2					,		sort_keys=_lowercase ) + '''\n'''
               f.write(_lowercase )
     else:
          UpperCAmelCase_     :				Any  =     BloomModel(_lowercase )

          UpperCAmelCase_     :				Tuple  =     os.listdir(_lowercase )
          UpperCAmelCase_     :				Union[str, Any]  =     sorted(filter(lambda _lowercase : s.startswith('''layer''' ) and "model_00" in s					,		_lowercase ) )

          UpperCAmelCase_     :				Any  =     None
          for i, file in enumerate(_lowercase ):
               UpperCAmelCase_     :				Optional[Any]  =     None
               for i in range(_lowercase ):
                    # load all TP files
                    UpperCAmelCase_     :				List[Any]  =     file.replace('''model_00'''					,		f'''model_0{i}''' )
                    UpperCAmelCase_     :				Optional[int]  =     torch.load(os.path.join(_lowercase					,		_lowercase )					,		map_location='''cpu''' )

                    # Rename keys in the transformers names
                    UpperCAmelCase_     :				str  =     list(temp.keys() )
                    for key in keys:
                         UpperCAmelCase_     :				Dict  =     temp.pop(_lowercase )

                    if tensors is None:
                         UpperCAmelCase_     :				int  =     temp
                    else:
                         for key in tensors.keys():
                              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                              if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                                   tensors[key] += temp[key]
                              else:
                                   # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                                   UpperCAmelCase_     :				Optional[int]  =     1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
                                   # We concatenate these weights accross TP ranks
                                   UpperCAmelCase_     :				List[str]  =     torch.cat([tensors[key], temp[key]]					,		dim=_lowercase )

            # Divide by the number of TP the weights we want to average
               for key in tensors.keys():
                    if any(key.endswith(_lowercase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
                         UpperCAmelCase_     :				Dict  =     tensors[key] / pretraining_tp

               UpperCAmelCase_     :				Tuple  =     model.load_state_dict(_lowercase					,		strict=_lowercase )
               assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
               if missing_keys is None:
                    UpperCAmelCase_     :				Union[str, Any]  =     set(other_keys.missing_keys )
               else:
                    UpperCAmelCase_     :				Dict  =     missing_keys.intersection(set(other_keys.missing_keys ) )

          assert not missing_keys, f'''The keys {missing_keys} are missing'''

          # Save pytorch-model
          os.makedirs(_lowercase					,		exist_ok=_lowercase )
          UpperCAmelCase_     :				str  =     pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
          UpperCAmelCase_     :				Dict  =     pytorch_dump_folder_path + '''/''' + CONFIG_NAME
          print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}''' )
          if config.torch_dtype is not None:
               UpperCAmelCase_     :				Optional[int]  =     model.to(config.torch_dtype )
          torch.save(model.state_dict()					,		_lowercase )
          print(f'''Save configuration file to {pytorch_config_dump_path}''' )
          with open(_lowercase					,		'''w'''					,		encoding='''utf-8''' ) as f:
               f.write(config.to_json_string() )


if __name__ == "__main__":
     __a							     =					argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '--bloom_checkpoint_path',
         default=None,
         type=str,
         required=True,
         help='Path to the Megatron-LM checkpoint path.',
     )
     parser.add_argument(
         '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
     )
     parser.add_argument(
         '--bloom_config_file',
         default='',
         type=str,
         help=(
             'An optional config json file corresponding to the pre-trained model. \n'
             'This specifies the model architecture.'
         ),
     )
     parser.add_argument(
         '--shard_model',
         action='store_true',
         help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
     )
     parser.add_argument(
         '--pretraining_tp',
         default=4,
         type=int,
         help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
     )
     __a							     =					parser.parse_args()
     convert_bloom_checkpoint_to_pytorch(
         args.bloom_checkpoint_path,
         args.bloom_config_file,
         args.pytorch_dump_folder_path,
         args.shard_model,
         args.pretraining_tp,
     )