

import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
      from transformers import LlamaTokenizerFast
except ImportError as e:
      warnings.warn(e)
      warnings.warn(
          """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
      )
      __lowercase				      =						None
__lowercase				      =						{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
__lowercase				      =						{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=1		,				SCREAMING_SNAKE_CASE=256   ):
      '''simple docstring'''







      return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      with open(SCREAMING_SNAKE_CASE		,				'''r'''   ) as f:
            return json.load(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      with open(SCREAMING_SNAKE_CASE		,				'''w'''   ) as f:
            json.dump(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=True   ):
      '''simple docstring'''







      os.makedirs(SCREAMING_SNAKE_CASE		,				exist_ok=SCREAMING_SNAKE_CASE   )
      A_							       =		os.path.join(SCREAMING_SNAKE_CASE		,				'''tmp'''   )
      os.makedirs(SCREAMING_SNAKE_CASE		,				exist_ok=SCREAMING_SNAKE_CASE   )

      A_							       =		read_json(os.path.join(SCREAMING_SNAKE_CASE		,				'''params.json'''   )   )
      A_							       =		NUM_SHARDS[model_size]
      A_							       =		params['''n_layers''']
      A_							       =		params['''n_heads''']
      A_							       =		n_heads // num_shards
      A_							       =		params['''dim''']
      A_							       =		dim // n_heads
      A_							       =		10_000.0
      A_							       =		1.0 / (base ** (torch.arange(0		,				SCREAMING_SNAKE_CASE		,				2   ).float() / dims_per_head))

      if "n_kv_heads" in params:
            A_							       =		params['''n_kv_heads''']  # for GQA / MQA
            A_							       =		n_heads_per_shard // num_key_value_heads
            A_							       =		dim // num_key_value_heads
      else:  # compatibility with other checkpoints
            A_							       =		n_heads
            A_							       =		n_heads_per_shard
            A_							       =		dim

      # permute for sliced rotary
      def permute(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE=n_heads		,				SCREAMING_SNAKE_CASE=dim		,				SCREAMING_SNAKE_CASE=dim   ):
            return w.view(SCREAMING_SNAKE_CASE		,				dima // n_heads // 2		,				2		,				SCREAMING_SNAKE_CASE   ).transpose(1		,				2   ).reshape(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )

      print(f"Fetching all parameters from the checkpoint at {input_base_path}."   )
      # Load weights
      if model_size == "7B":
            # Not sharded
            # (The sharded implementation would also work, but this is simpler.)
            A_							       =		torch.load(os.path.join(SCREAMING_SNAKE_CASE		,				'''consolidated.00.pth'''   )		,				map_location='''cpu'''   )
      else:
            # Sharded
            A_							       =		[
                torch.load(os.path.join(SCREAMING_SNAKE_CASE		,				f"consolidated.{i:02d}.pth"   )		,				map_location='''cpu'''   )
                for i in range(SCREAMING_SNAKE_CASE   )
            ]
      A_							       =		0
      A_							       =		{'''weight_map''': {}}
      for layer_i in range(SCREAMING_SNAKE_CASE   ):
            A_							       =		f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
            if model_size == "7B":
                  # Unsharded
                  A_							       =		{
                      f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
                          loaded[f"layers.{layer_i}.attention.wq.weight"]   ),
                      f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
                          loaded[f"layers.{layer_i}.attention.wk.weight"]   ),
                      f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
                      f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
                      f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
                      f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
                      f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
                      f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
                      f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
                  }
            else:
                  # Sharded
                  # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                  # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                  # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                  A_							       =		{
                      f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
                          f"layers.{layer_i}.attention_norm.weight"
                      ].clone(),
                      f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
                          f"layers.{layer_i}.ffn_norm.weight"
                      ].clone(),
                  }
                  A_							       =		permute(
                      torch.cat(
                          [
                              loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                              for i in range(SCREAMING_SNAKE_CASE   )
                          ]		,				dim=0		,				).reshape(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )   )
                  A_							       =		permute(
                      torch.cat(
                          [
                              loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                                  SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                              for i in range(SCREAMING_SNAKE_CASE   )
                          ]		,				dim=0		,				).reshape(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				)
                  A_							       =		torch.cat(
                      [
                          loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                              SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                          for i in range(SCREAMING_SNAKE_CASE   )
                      ]		,				dim=0		,				).reshape(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )

                  A_							       =		torch.cat(
                      [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=1   )
                  A_							       =		torch.cat(
                      [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=0   )
                  A_							       =		torch.cat(
                      [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=1   )
                  A_							       =		torch.cat(
                      [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=0   )

            A_							       =		inv_freq
            for k, v in state_dict.items():
                  A_							       =		filename
                  param_count += v.numel()
            torch.save(SCREAMING_SNAKE_CASE		,				os.path.join(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )   )

      A_							       =		f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
      if model_size == "7B":
            # Unsharded
            A_							       =		{
                '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
                '''model.norm.weight''': loaded['''norm.weight'''],
                '''lm_head.weight''': loaded['''output.weight'''],
            }
      else:
            A_							       =		{
                '''model.norm.weight''': loaded[0]['''norm.weight'''],
                '''model.embed_tokens.weight''': torch.cat(
                    [loaded[i]['''tok_embeddings.weight'''] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=1   ),
                '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(SCREAMING_SNAKE_CASE   )]		,				dim=0   ),
            }

      for k, v in state_dict.items():
            A_							       =		filename
            param_count += v.numel()
      torch.save(SCREAMING_SNAKE_CASE		,				os.path.join(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )   )

      # Write configs
      A_							       =		{'''total_size''': param_count * 2}
      write_json(SCREAMING_SNAKE_CASE		,				os.path.join(SCREAMING_SNAKE_CASE		,				'''pytorch_model.bin.index.json'''   )   )
      A_							       =		params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
      A_							       =		params['''multiple_of'''] if '''multiple_of''' in params else 256
      A_							       =		LlamaConfig(
          hidden_size=SCREAMING_SNAKE_CASE		,				intermediate_size=compute_intermediate_size(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )		,				num_attention_heads=params['''n_heads''']		,				num_hidden_layers=params['''n_layers''']		,				rms_norm_eps=params['''norm_eps''']		,				num_key_value_heads=SCREAMING_SNAKE_CASE		,				)
      config.save_pretrained(SCREAMING_SNAKE_CASE   )

      # Make space so we can load the model properly now.
      del state_dict
      del loaded
      gc.collect()

      print('''Loading the checkpoint in a Llama model.'''   )
      A_							       =		LlamaForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE		,				torch_dtype=torch.floataa		,				low_cpu_mem_usage=SCREAMING_SNAKE_CASE   )
      # Avoid saving this as part of the config.
      del model.config._name_or_path

      print('''Saving in the Transformers format.'''   )
      model.save_pretrained(SCREAMING_SNAKE_CASE		,				safe_serialization=SCREAMING_SNAKE_CASE   )
      shutil.rmtree(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
      print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."   )
      A_							       =		tokenizer_class(SCREAMING_SNAKE_CASE   )
      tokenizer.save_pretrained(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		argparse.ArgumentParser()
      parser.add_argument(
          '''--input_dir'''		,				help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''		,				)
      parser.add_argument(
          '''--model_size'''		,				choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']		,				)
      parser.add_argument(
          '''--output_dir'''		,				help='''Location to write HF model and tokenizer'''		,				)
      parser.add_argument('''--safe_serialization'''		,				type=SCREAMING_SNAKE_CASE		,				help='''Whether or not to save using `safetensors`.'''   )
      A_							       =		parser.parse_args()
      if args.model_size != "tokenizer_only":
            write_model(
                model_path=args.output_dir		,				input_base_path=os.path.join(args.input_dir		,				args.model_size   )		,				model_size=args.model_size		,				safe_serialization=args.safe_serialization		,				)
      A_							       =		os.path.join(args.input_dir		,				'''tokenizer.model'''   )
      write_tokenizer(args.output_dir		,				SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
      main()








import unittest
from typing import Dict, List, Optional, Union

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import BridgeTowerImageProcessor

class    _lowercase				(					unittest.TestCase			):

   def __init__(				self	:    List[str]			,						lowerCamelCase__	:    List[str]			,						lowerCamelCase__	:    bool = True			,						lowerCamelCase__	:    Dict[str, int] = None			,						lowerCamelCase__	:    int = 3_2			,						lowerCamelCase__	:    bool = True			,						lowerCamelCase__	:    Union[int, float] = 1 / 2_5_5			,						lowerCamelCase__	:    bool = True			,						lowerCamelCase__	:    bool = True			,						lowerCamelCase__	:    Optional[Union[float, List[float]]] = [0.48145466, 0.4578275, 0.40821073]			,						lowerCamelCase__	:    Optional[Union[float, List[float]]] = [0.26862954, 0.26130258, 0.27577711]			,						lowerCamelCase__	:    bool = True			,						lowerCamelCase__	:    List[str]=7			,						lowerCamelCase__	:    int=3_0			,						lowerCamelCase__	:    List[Any]=4_0_0			,						lowerCamelCase__	:    int=3			,						) ->			Optional[Any]:
         """simple docstring"""





         A_							       =		parent
         A_							       =		do_resize
         A_							       =		size if size is not None else {'''shortest_edge''': 2_8_8}
         A_							       =		size_divisor
         A_							       =		do_rescale
         A_							       =		rescale_factor
         A_							       =		do_normalize
         A_							       =		do_center_crop
         A_							       =		image_mean
         A_							       =		image_std
         A_							       =		do_pad
         A_							       =		batch_size
         A_							       =		num_channels
         A_							       =		min_resolution
         A_							       =		max_resolution

   def     UpperCamelCase					(				self	:    str       ) ->			int:
         """simple docstring"""





         return {
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_normalize": self.do_normalize,
             "do_resize": self.do_resize,
             "size": self.size,
             "size_divisor": self.size_divisor,
         }





   def     UpperCamelCase					(				self	:    List[str]			,						lowerCamelCase__	:    Optional[Any]			,						lowerCamelCase__	:    int=False       ) ->			Dict:
         """simple docstring"""





         if not batched:
               A_							       =		self.size['''shortest_edge''']
               A_							       =		image_inputs[0]
               if isinstance(lowerCamelCase__			,						Image.Image       ):
                     A_       ,A_							       =		image.size
               else:
                     A_       ,A_							       =		image.shape[1], image.shape[2]
               A_							       =		size / min(lowerCamelCase__			,						lowerCamelCase__       )
               if h < w:
                     A_       ,A_							       =		size, scale * w
               else:
                     A_       ,A_							       =		scale * h, size

               A_							       =		int((1_3_3_3 / 8_0_0) * size       )
               if max(lowerCamelCase__			,						lowerCamelCase__       ) > max_size:
                     A_							       =		max_size / max(lowerCamelCase__			,						lowerCamelCase__       )
                     A_							       =		newh * scale
                     A_							       =		neww * scale

               A_       ,A_							       =		int(newh + 0.5       ), int(neww + 0.5       )
               A_       ,A_							       =		(
                   newh // self.size_divisor * self.size_divisor,
                   neww // self.size_divisor * self.size_divisor,
               )

         else:
               A_							       =		[]
               for image in image_inputs:
                     A_       ,A_							       =		self.get_expected_values([image]       )
                     expected_values.append((expected_height, expected_width)       )
               A_							       =		max(lowerCamelCase__			,						key=lambda lowerCamelCase__       : item[0]       )[0]
               A_							       =		max(lowerCamelCase__			,						key=lambda lowerCamelCase__       : item[1]       )[1]

         return expected_height, expected_width



@require_torch
@require_vision
class    _lowercase				(					__lowerCamelCase,unittest.TestCase			):
   _lowercase     :       List[str]  					=    BridgeTowerImageProcessor if is_vision_available() else None

   def     UpperCamelCase					(				self	:    int       ) ->			Tuple:
         """simple docstring"""





         A_							       =		BridgeTowerImageProcessingTester(self       )

   @property
   def     UpperCamelCase					(				self	:    str       ) ->			Optional[int]:
         """simple docstring"""





         return self.image_processor_tester.prepare_image_processor_dict()

   def     UpperCamelCase					(				self	:    Tuple       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		self.image_processing_class(**self.image_processor_dict       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''image_mean'''       )       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''image_std'''       )       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''do_normalize'''       )       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''do_resize'''       )       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''size'''       )       )
         self.assertTrue(hasattr(lowerCamelCase__			,						'''size_divisor'''       )       )

   def     UpperCamelCase					(				self	:    Optional[int]       ) ->			str:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    Optional[int]       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		self.image_processing_class(**self.image_processor_dict       )
         # create random PIL images
         A_							       =		prepare_image_inputs(self.image_processor_tester			,						equal_resolution=lowerCamelCase__       )
         for image in image_inputs:
               self.assertIsInstance(lowerCamelCase__			,						Image.Image       )

         # Test not batched input
         A_							       =		image_processing(image_inputs[0]			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

         # Test batched
         A_							       =		image_processing(lowerCamelCase__			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__			,						batched=lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )			,						)

   def     UpperCamelCase					(				self	:    str       ) ->			Optional[int]:
         """simple docstring"""





         A_							       =		self.image_processing_class(**self.image_processor_dict       )
         # create random numpy tensors
         A_							       =		prepare_image_inputs(self.image_processor_tester			,						equal_resolution=lowerCamelCase__			,						numpify=lowerCamelCase__       )
         for image in image_inputs:
               self.assertIsInstance(lowerCamelCase__			,						np.ndarray       )

         # Test not batched input
         A_							       =		image_processing(image_inputs[0]			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

         # Test batched
         A_							       =		image_processing(lowerCamelCase__			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__			,						batched=lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )			,						)





   def     UpperCamelCase					(				self	:    Union[str, Any]       ) ->			int:
         """simple docstring"""





         A_							       =		self.image_processing_class(**self.image_processor_dict       )
         # create random PyTorch tensors
         A_							       =		prepare_image_inputs(self.image_processor_tester			,						equal_resolution=lowerCamelCase__			,						torchify=lowerCamelCase__       )
         for image in image_inputs:
               self.assertIsInstance(lowerCamelCase__			,						torch.Tensor       )

         # Test not batched input
         A_							       =		image_processing(image_inputs[0]			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

         # Test batched
         A_							       =		image_processing(lowerCamelCase__			,						return_tensors='''pt'''       ).pixel_values

         A_       ,A_							       =		self.image_processor_tester.get_expected_values(lowerCamelCase__			,						batched=lowerCamelCase__       )
         self.assertEqual(
             encoded_images.shape			,						(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 expected_height,
                 expected_width,
             )			,						)





