



'''simple docstring'''





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel






class __A     :



 '''simple docstring'''







 def __init__(self					,  A					,  A=13					,  A=7					,  A=True					,  A=True					,  A=False					,  A=True					,  A=99					,  A=32					,  A=5					,  A=4					,  A=37					,  A="gelu"					,  A=0.1					,  A=0.1					,  A=512					,  A=16					,  A=2					,  A=0.02					,  A=3					,  A=4					,  A=None					,  )       ->	str:


      """simple docstring"""



      _a           =    parent
      _a           =    batch_size
      _a           =    seq_length
      _a           =    is_training
      _a           =    use_input_mask
      _a           =    use_token_type_ids
      _a           =    use_labels
      _a           =    vocab_size
      _a           =    hidden_size
      _a           =    num_hidden_layers
      _a           =    num_attention_heads
      _a           =    intermediate_size
      _a           =    hidden_act
      _a           =    hidden_dropout_prob
      _a           =    attention_probs_dropout_prob
      _a           =    max_position_embeddings
      _a           =    type_vocab_size
      _a           =    type_sequence_label_size
      _a           =    initializer_range
      _a           =    num_labels
      _a           =    num_choices
      _a           =    scope



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      _a           =    ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size			)

      _a           =    None
      if self.use_input_mask:
           _a           =    random_attention_mask([self.batch_size, self.seq_length]			)

      _a           =    None
      if self.use_token_type_ids:
           _a           =    ids_tensor([self.batch_size, self.seq_length]					,  self.type_vocab_size			)

      _a           =    None
      _a           =    None
      _a           =    None
      if self.use_labels:
           _a           =    ids_tensor([self.batch_size]					,  self.type_sequence_label_size			)
           _a           =    ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels			)
           _a           =    ids_tensor([self.batch_size]					,  self.num_choices			)

      _a           =    self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      return OpenLlamaConfig(
          vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  type_vocab_size=self.type_vocab_size					,  is_decoder=A					,  initializer_range=self.initializer_range					,  use_stable_embedding=A					,  )



 def    a__    (self					,  A					,  A					,  A					,  A					,  A					,  A					,  A			)       ->	Any:


      """simple docstring"""



      _a           =    OpenLlamaModel(config=A			)
      model.to(A			)
      model.eval()
      _a           =    model(A					,  attention_mask=A			)
      _a           =    model(A			)
      self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)			)



 def    a__    (self					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  )       ->	Any:


      """simple docstring"""



      _a           =    True
      _a           =    OpenLlamaModel(A			)
      model.to(A			)
      model.eval()
      _a           =    model(
          A					,  attention_mask=A					,  encoder_hidden_states=A					,  encoder_attention_mask=A					,  )
      _a           =    model(
          A					,  attention_mask=A					,  encoder_hidden_states=A					,  )
      _a           =    model(A					,  attention_mask=A			)
      self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)			)



 def    a__    (self					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  )       ->	Tuple:


      """simple docstring"""



      _a           =    OpenLlamaForCausalLM(config=A			)
      model.to(A			)
      model.eval()
      _a           =    model(A					,  attention_mask=A					,  labels=A			)
      self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)			)



 def    a__    (self					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  A					,  )       ->	Dict:


      """simple docstring"""



      _a           =    True
      _a           =    True
      _a           =    OpenLlamaForCausalLM(config=A			)
      model.to(A			)
      model.eval()

      # first forward pass
      _a           =    model(
          A					,  attention_mask=A					,  encoder_hidden_states=A					,  encoder_attention_mask=A					,  use_cache=A					,  )
      _a           =    outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      _a           =    ids_tensor((self.batch_size, 3)					,  config.vocab_size			)
      _a           =    ids_tensor((self.batch_size, 3)					,  vocab_size=2			)

      # append to next input_ids and
      _a           =    torch.cat([input_ids, next_tokens]					,  dim=-1			)
      _a           =    torch.cat([input_mask, next_mask]					,  dim=-1			)

      _a           =    model(
          A					,  attention_mask=A					,  encoder_hidden_states=A					,  encoder_attention_mask=A					,  output_hidden_states=A					,  )['''hidden_states'''][0]
      _a           =    model(
          A					,  attention_mask=A					,  encoder_hidden_states=A					,  encoder_attention_mask=A					,  past_key_values=A					,  output_hidden_states=A					,  )['''hidden_states'''][0]

      # select random slice
      _a           =    ids_tensor((1,)					,  output_from_past.shape[-1]			).item()
      _a           =    output_from_no_past[:, -3:, random_slice_idx].detach()
      _a           =    output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(A					,  A					,  atol=1E-3			)			)



 def    a__    (self			)       ->	Optional[Any]:


      """simple docstring"""



      _a           =    self.prepare_config_and_inputs()
      (
          (
          _a
      )    ,       (
          _a
      )    ,       (
          _a
      )    ,       (
          _a
      )    ,       (
          _a
      )    ,       (
          _a
      )    ,       (
          _a
      )    ,       
      )           =    config_and_inputs
      _a           =    {'''input_ids''': input_ids, '''attention_mask''': input_mask}
      return config, inputs_dict






@require_torch
class __A     (    A  ,  A  ,  A  ,  unittest.TestCase		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[int]		=			(
     (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
 )
 __lowerCamelCase       :			Any		=			(OpenLlamaForCausalLM,) if is_torch_available() else ()
 __lowerCamelCase       :			List[Any]		=			(
     {
         'feature-extraction': OpenLlamaModel,
         'text-classification': OpenLlamaForSequenceClassification,
         'text-generation': OpenLlamaForCausalLM,
         'zero-shot': OpenLlamaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowerCamelCase       :			List[str]		=			False
 __lowerCamelCase       :			List[str]		=			False



 def    a__    (self			)       ->	Tuple:


      """simple docstring"""



      _a           =    OpenLlamaModelTester(self			)
      _a           =    ConfigTester(self					,  config_class=A					,  hidden_size=37			)



 def    a__    (self			)       ->	List[str]:


      """simple docstring"""



      self.config_tester.run_common_tests()



 def    a__    (self			)       ->	Union[str, Any]:


      """simple docstring"""



      _a           =    self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*A			)



 def    a__    (self			)       ->	str:


      """simple docstring"""



      _a           =    self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
           _a           =    type
           self.model_tester.create_and_check_model(*A			)



 def    a__    (self			)       ->	Any:


      """simple docstring"""



      _a    ,       _a           =    self.model_tester.prepare_config_and_inputs_for_common()
      _a           =    3
      _a           =    input_dict['''input_ids''']
      _a           =    input_ids.ne(1			).to(A			)
      _a           =    ids_tensor([self.model_tester.batch_size]					,  self.model_tester.type_sequence_label_size			)
      _a           =    OpenLlamaForSequenceClassification(A			)
      model.to(A			)
      model.eval()
      _a           =    model(A					,  attention_mask=A					,  labels=A			)
      self.assertEqual(result.logits.shape					,  (self.model_tester.batch_size, self.model_tester.num_labels)			)



 def    a__    (self			)       ->	Dict:


      """simple docstring"""



      _a    ,       _a           =    self.model_tester.prepare_config_and_inputs_for_common()
      _a           =    3
      _a           =    '''single_label_classification'''
      _a           =    input_dict['''input_ids''']
      _a           =    input_ids.ne(1			).to(A			)
      _a           =    ids_tensor([self.model_tester.batch_size]					,  self.model_tester.type_sequence_label_size			)
      _a           =    OpenLlamaForSequenceClassification(A			)
      model.to(A			)
      model.eval()
      _a           =    model(A					,  attention_mask=A					,  labels=A			)
      self.assertEqual(result.logits.shape					,  (self.model_tester.batch_size, self.model_tester.num_labels)			)



 def    a__    (self			)       ->	Optional[Any]:


      """simple docstring"""



      _a    ,       _a           =    self.model_tester.prepare_config_and_inputs_for_common()
      _a           =    3
      _a           =    '''multi_label_classification'''
      _a           =    input_dict['''input_ids''']
      _a           =    input_ids.ne(1			).to(A			)
      _a           =    ids_tensor(
          [self.model_tester.batch_size, config.num_labels]					,  self.model_tester.type_sequence_label_size			).to(torch.float			)
      _a           =    OpenLlamaForSequenceClassification(A			)
      model.to(A			)
      model.eval()
      _a           =    model(A					,  attention_mask=A					,  labels=A			)
      self.assertEqual(result.logits.shape					,  (self.model_tester.batch_size, self.model_tester.num_labels)			)



 @unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''			)
 def    a__    (self			)       ->	Optional[Any]:


      """simple docstring"""



      pass



 @parameterized.expand([('''linear''',), ('''dynamic''',)]			)
 def    a__    (self					,  A			)       ->	Optional[int]:


      """simple docstring"""



      _a    ,       _a           =    self.model_tester.prepare_config_and_inputs_for_common()
      _a           =    ids_tensor([1, 10]					,  config.vocab_size			)
      _a           =    ids_tensor([1, int(config.max_position_embeddings * 1.5			)]					,  config.vocab_size			)

      set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
      _a           =    OpenLlamaModel(A			)
      original_model.to(A			)
      original_model.eval()
      _a           =    original_model(A			).last_hidden_state
      _a           =    original_model(A			).last_hidden_state

      set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
      _a           =    {'''type''': scaling_type, '''factor''': 10.0}
      _a           =    OpenLlamaModel(A			)
      scaled_model.to(A			)
      scaled_model.eval()
      _a           =    scaled_model(A			).last_hidden_state
      _a           =    scaled_model(A			).last_hidden_state

      # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
      # maximum sequence length, so the outputs for the short input should match.
      if scaling_type == "dynamic":
           self.assertTrue(torch.allclose(A					,  A					,  atol=1E-5			)			)
      else:
           self.assertFalse(torch.allclose(A					,  A					,  atol=1E-5			)			)

      # The output should be different for long inputs
      self.assertFalse(torch.allclose(A					,  A					,  atol=1E-5			)			)











"""simple docstring"""





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__lowerCamelCase							=  [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

__lowerCamelCase							=  [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]



def   a    (     __UpperCAmelCase      :    Any      ,   __UpperCAmelCase      :    Any  )    ->							Any:
						__magic_name__:	List[str]							=       {
						    """word_embeddings.weight""": """word_embeddings.weight""",
						    """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
						    """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
						    """weight""": """ln_f.weight""",
						    """bias""": """ln_f.bias""",
						}

						if key in layer_rename_map:
												return layer_rename_map[key]

						# Handle transformer blocks
						__magic_name__:	Any							=       int(re.match(R""".*layer_(\d*).*"""      ,   __UpperCAmelCase  )[1]  )
						layer_number -= 3
						return f'h.{layer_number}.' + key



def   a    (     __UpperCAmelCase      :    List[str]  )    ->							Union[str, Any]:
						if dtype == torch.bool:
												return 1 / 8
						__magic_name__:	Optional[int]							=       re.search(R"""[^\d](\d+)$"""      ,   str(__UpperCAmelCase  )  )
						if bit_search is None:
												raise ValueError(f'`dtype` is not a valid dtype: {dtype}.'  )
						__magic_name__:	Dict							=       int(bit_search.groups()[0]  )
						return bit_size // 8



def   a    (     __UpperCAmelCase      :    Union[str, Any]      ,   __UpperCAmelCase      :    Optional[int]      ,   __UpperCAmelCase      :    List[Any]      ,   __UpperCAmelCase      :    Optional[int]      ,   __UpperCAmelCase      :    Tuple  )    ->							List[str]:
						# Construct model
						if bloom_config_file == "":
												__magic_name__:	Optional[Any]							=       BloomConfig()
						else:
												__magic_name__:	Tuple							=       BloomConfig.from_json_file(__UpperCAmelCase  )

						if shard_model:
												__magic_name__:	Union[str, Any]							=       os.listdir(__UpperCAmelCase  )
												__magic_name__:	Optional[int]							=       sorted(filter(lambda __UpperCAmelCase  : s.startswith("""layer"""  ) and "model_00" in s      ,   __UpperCAmelCase  )  )

												__magic_name__:	str							=       {"""weight_map""": {}, """metadata""": {}}
												__magic_name__:	Optional[Any]							=       0

												__magic_name__:	Dict							=       None

												__magic_name__:	Optional[int]							=       BloomConfig()

												for j, file in enumerate(__UpperCAmelCase  ):
																		print("""Processing file: {}""".format(__UpperCAmelCase  )  )
																		__magic_name__:	Tuple							=       None

																		for i in range(__UpperCAmelCase  ):
																								# load all TP files
																								__magic_name__:	int							=       file.replace("""model_00"""      ,   f'model_0{i}'  )
																								__magic_name__:	Optional[Any]							=       torch.load(os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )      ,   map_location="""cpu"""  )

																								# Rename keys in the transformers names
																								__magic_name__:	str							=       list(temp.keys()  )
																								for key in keys:
																														__magic_name__:	Any							=       temp.pop(__UpperCAmelCase  )

																								if tensors is None:
																														__magic_name__:	Union[str, Any]							=       temp
																								else:
																														for key in tensors.keys():
																																				if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__magic_name__:	int							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																										# We concatenate these weights accross TP ranks
																																										__magic_name__:	int							=       torch.cat([tensors[key], temp[key]]      ,   dim=__UpperCAmelCase  )

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																														__magic_name__:	Union[str, Any]							=       tensors[key] / pretraining_tp
																		torch.save(
																		    __UpperCAmelCase      ,   os.path.join(
																		        __UpperCAmelCase      ,   """pytorch_model_{}-of-{}.bin""".format(str(j + 1  ).zfill(5  )      ,   str(len(__UpperCAmelCase  )  ).zfill(5  )  )      ,   )      ,   )

																		for key in tensors.keys():
																								__magic_name__:	Optional[Any]							=       tensors[key]
																								total_size += value.numel() * get_dtype_size(value.dtype  )
																								if key not in index_dict["weight_map"]:
																														__magic_name__:	Union[str, Any]							=       """pytorch_model_{}-of-{}.bin""".format(
																														    str(j + 1  ).zfill(5  )      ,   str(len(__UpperCAmelCase  )  ).zfill(5  )  )

												__magic_name__:	Tuple							=       BloomConfig()
												__magic_name__:	Optional[int]							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
												__magic_name__:	Optional[int]							=       total_size
												with open(__UpperCAmelCase      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		f.write(config.to_json_string()  )
												with open(os.path.join(__UpperCAmelCase      ,   WEIGHTS_NAME + """.index.json"""  )      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		__magic_name__:	str							=       json.dumps(__UpperCAmelCase      ,   indent=2      ,   sort_keys=__UpperCAmelCase  ) + """\n"""
																		f.write(__UpperCAmelCase  )
						else:
												__magic_name__:	Optional[int]							=       BloomModel(__UpperCAmelCase  )

												__magic_name__:	List[str]							=       os.listdir(__UpperCAmelCase  )
												__magic_name__:	Optional[Any]							=       sorted(filter(lambda __UpperCAmelCase  : s.startswith("""layer"""  ) and "model_00" in s      ,   __UpperCAmelCase  )  )

												__magic_name__:	Any							=       None
												for i, file in enumerate(__UpperCAmelCase  ):
																		__magic_name__:	List[Any]							=       None
																		for i in range(__UpperCAmelCase  ):
																								# load all TP files
																								__magic_name__:	Any							=       file.replace("""model_00"""      ,   f'model_0{i}'  )
																								__magic_name__:	Dict							=       torch.load(os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )      ,   map_location="""cpu"""  )

																								# Rename keys in the transformers names
																								__magic_name__:	Optional[Any]							=       list(temp.keys()  )
																								for key in keys:
																														__magic_name__:	Optional[Any]							=       temp.pop(__UpperCAmelCase  )

																								if tensors is None:
																														__magic_name__:	Tuple							=       temp
																								else:
																														for key in tensors.keys():
																																				# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																				if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__magic_name__:	Tuple							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																										# We concatenate these weights accross TP ranks
																																										__magic_name__:	Dict							=       torch.cat([tensors[key], temp[key]]      ,   dim=__UpperCAmelCase  )

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																														__magic_name__:	Optional[int]							=       tensors[key] / pretraining_tp

																		__magic_name__:	Union[str, Any]							=       model.load_state_dict(__UpperCAmelCase      ,   strict=__UpperCAmelCase  )
																		assert not other_keys.unexpected_keys, f'The keys {other_keys.unexpected_keys} are unexpected'
																		if missing_keys is None:
																								__magic_name__:	Optional[Any]							=       set(other_keys.missing_keys  )
																		else:
																								__magic_name__:	Tuple							=       missing_keys.intersection(set(other_keys.missing_keys  )  )

												assert not missing_keys, f'The keys {missing_keys} are missing'

												# Save pytorch-model
												os.makedirs(__UpperCAmelCase      ,   exist_ok=__UpperCAmelCase  )
												__magic_name__:	int							=       pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
												__magic_name__:	List[str]							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
												print(f'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'  )
												if config.torch_dtype is not None:
																		__magic_name__:	Optional[Any]							=       model.to(config.torch_dtype  )
												torch.save(model.state_dict()      ,   __UpperCAmelCase  )
												print(f'Save configuration file to {pytorch_config_dump_path}'  )
												with open(__UpperCAmelCase      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		f.write(config.to_json_string()  )


if __name__ == "__main__":
							__lowerCamelCase							=  argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bloom_checkpoint_path',
							    default=None,
							    type=str,
							    required=True,
							    help='Path to the Megatron-LM checkpoint path.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							parser.add_argument(
							    '--bloom_config_file',
							    default='',
							    type=str,
							    help=(
							        'An optional config json file corresponding to the pre-trained model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--shard_model',
							    action='store_true',
							    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
							)
							parser.add_argument(
							    '--pretraining_tp',
							    default=4,
							    type=int,
							    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
							)
							__lowerCamelCase							=  parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)



