


'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
      from transformers import LlamaTokenizerFast
except ImportError as e:
      warnings.warn(e)
      warnings.warn(
          """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
      )
      _UpperCamelCase 							=  None
_UpperCamelCase 							=  {
    """7B""": 1_1008,
    """13B""": 1_3824,
    """30B""": 1_7920,
    """65B""": 2_2016,
    """70B""": 2_8672,
}
_UpperCamelCase 							=  {
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE=1       ,					SCREAMING_SNAKE_CASE=256							):




     '''simple docstring'''





     return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3							)							) + multiple_of - 1) // multiple_of)


def 		_lowercase (SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     with open(SCREAMING_SNAKE_CASE       ,					"r"							) as f:
          return json.load(SCREAMING_SNAKE_CASE							)


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     with open(SCREAMING_SNAKE_CASE       ,					"w"							) as f:
          json.dump(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE=True							):




     '''simple docstring'''





     os.makedirs(SCREAMING_SNAKE_CASE       ,					exist_ok=SCREAMING_SNAKE_CASE							)
     __A	:       str      =							os.path.join(SCREAMING_SNAKE_CASE       ,					"tmp"							)
     os.makedirs(SCREAMING_SNAKE_CASE       ,					exist_ok=SCREAMING_SNAKE_CASE							)

     __A	:       str      =							read_json(os.path.join(SCREAMING_SNAKE_CASE       ,					"params.json"							)							)
     __A	:       Dict      =							NUM_SHARDS[model_size]
     __A	:       Any      =							params["n_layers"]
     __A	:       List[str]      =							params["n_heads"]
     __A	:       Any      =							n_heads // num_shards
     __A	:       Optional[Any]      =							params["dim"]
     __A	:       str      =							dim // n_heads
     __A	:       Optional[Any]      =							10000.0
     __A	:       Tuple      =							1.0 / (base ** (torch.arange(0       ,					SCREAMING_SNAKE_CASE       ,					2							).float() / dims_per_head))

     if "n_kv_heads" in params:
          __A	:       List[str]      =							params["n_kv_heads"]  # for GQA / MQA
          __A	:       Optional[int]      =							n_heads_per_shard // num_key_value_heads
          __A	:       List[Any]      =							dim // num_key_value_heads
     else:  # compatibility with other checkpoints
          __A	:       List[str]      =							n_heads
          __A	:       Optional[int]      =							n_heads_per_shard
          __A	:       List[Any]      =							dim

     # permute for sliced rotary
     def permute(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE=n_heads       ,					SCREAMING_SNAKE_CASE=dim       ,					SCREAMING_SNAKE_CASE=dim							):
          return w.view(SCREAMING_SNAKE_CASE       ,					dima // n_heads // 2       ,					2       ,					SCREAMING_SNAKE_CASE							).transpose(1       ,					2							).reshape(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)

     print(f"Fetching all parameters from the checkpoint at {input_base_path}."							)
     # Load weights
     if model_size == "7B":
          # Not sharded
          # (The sharded implementation would also work, but this is simpler.)
          __A	:       Union[str, Any]      =							torch.load(os.path.join(SCREAMING_SNAKE_CASE       ,					"consolidated.00.pth"							)       ,					map_location="cpu"							)
     else:
          # Sharded
          __A	:       Tuple      =							[
              torch.load(os.path.join(SCREAMING_SNAKE_CASE       ,					f"consolidated.{i:02d}.pth"							)       ,					map_location="cpu"							)
              for i in range(SCREAMING_SNAKE_CASE							)
          ]
     __A	:       Union[str, Any]      =							0
     __A	:       str      =							{"weight_map": {}}
     for layer_i in range(SCREAMING_SNAKE_CASE							):
          __A	:       str      =							f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
          if model_size == "7B":
               # Unsharded
               __A	:       int      =							{
                   f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
                       loaded[f"layers.{layer_i}.attention.wq.weight"]							),
                   f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
                       loaded[f"layers.{layer_i}.attention.wk.weight"]							),
                   f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
                   f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
                   f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
                   f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
                   f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
                   f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
                   f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
               }
          else:
               # Sharded
               # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
               # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
               # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

               __A	:       Optional[Any]      =							{
                   f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
                       f"layers.{layer_i}.attention_norm.weight"
                   ].clone(),
                   f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
                       f"layers.{layer_i}.ffn_norm.weight"
                   ].clone(),
               }
               __A	:       Tuple      =							permute(
                   torch.cat(
                       [
                           loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                           for i in range(SCREAMING_SNAKE_CASE							)
                       ]       ,					dim=0       ,					).reshape(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)							)
               __A	:       str      =							permute(
                   torch.cat(
                       [
                           loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                               SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                           for i in range(SCREAMING_SNAKE_CASE							)
                       ]       ,					dim=0       ,					).reshape(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					)
               __A	:       Optional[int]      =							torch.cat(
                   [
                       loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                           SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                       for i in range(SCREAMING_SNAKE_CASE							)
                   ]       ,					dim=0       ,					).reshape(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)

               __A	:       Union[str, Any]      =							torch.cat(
                   [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=1							)
               __A	:       Union[str, Any]      =							torch.cat(
                   [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=0							)
               __A	:       int      =							torch.cat(
                   [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=1							)
               __A	:       Dict      =							torch.cat(
                   [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=0							)

          __A	:       Optional[int]      =							inv_freq
          for k, v in state_dict.items():
               __A	:       Optional[int]      =							filename
               param_count += v.numel()
          torch.save(SCREAMING_SNAKE_CASE       ,					os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)							)

     __A	:       Union[str, Any]      =							f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
     if model_size == "7B":
          # Unsharded
          __A	:       Any      =							{
              "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
              "model.norm.weight": loaded["norm.weight"],
              "lm_head.weight": loaded["output.weight"],
          }
     else:
          __A	:       List[str]      =							{
              "model.norm.weight": loaded[0]["norm.weight"],
              "model.embed_tokens.weight": torch.cat(
                  [loaded[i]["tok_embeddings.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=1							),
              "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(SCREAMING_SNAKE_CASE							)]       ,					dim=0							),
          }

     for k, v in state_dict.items():
          __A	:       Dict      =							filename
          param_count += v.numel()
     torch.save(SCREAMING_SNAKE_CASE       ,					os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)							)

     # Write configs
     __A	:       Optional[int]      =							{"total_size": param_count * 2}
     write_json(SCREAMING_SNAKE_CASE       ,					os.path.join(SCREAMING_SNAKE_CASE       ,					"pytorch_model.bin.index.json"							)							)
     __A	:       List[Any]      =							params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
     __A	:       int      =							params["multiple_of"] if "multiple_of" in params else 256
     __A	:       Dict      =							LlamaConfig(
         hidden_size=SCREAMING_SNAKE_CASE       ,					intermediate_size=compute_intermediate_size(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)       ,					num_attention_heads=params["n_heads"]       ,					num_hidden_layers=params["n_layers"]       ,					rms_norm_eps=params["norm_eps"]       ,					num_key_value_heads=SCREAMING_SNAKE_CASE       ,					)
     config.save_pretrained(SCREAMING_SNAKE_CASE							)

     # Make space so we can load the model properly now.
     del state_dict
     del loaded
     gc.collect()

     print("Loading the checkpoint in a Llama model."							)
     __A	:       Union[str, Any]      =							LlamaForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE       ,					torch_dtype=torch.floataa       ,					low_cpu_mem_usage=SCREAMING_SNAKE_CASE							)
     # Avoid saving this as part of the config.
     del model.config._name_or_path

     print("Saving in the Transformers format."							)
     model.save_pretrained(SCREAMING_SNAKE_CASE       ,					safe_serialization=SCREAMING_SNAKE_CASE							)
     shutil.rmtree(SCREAMING_SNAKE_CASE							)


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       Union[str, Any]      =							LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
     print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."							)
     __A	:       Any      =							tokenizer_class(SCREAMING_SNAKE_CASE							)
     tokenizer.save_pretrained(SCREAMING_SNAKE_CASE							)


def 		_lowercase ():




     '''simple docstring'''





     __A	:       Tuple      =							argparse.ArgumentParser()
     parser.add_argument(
         "--input_dir"       ,					help="Location of LLaMA weights, which contains tokenizer.model and model folders"       ,					)
     parser.add_argument(
         "--model_size"       ,					choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"]       ,					)
     parser.add_argument(
         "--output_dir"       ,					help="Location to write HF model and tokenizer"       ,					)
     parser.add_argument("--safe_serialization"       ,					type=SCREAMING_SNAKE_CASE       ,					help="Whether or not to save using `safetensors`."							)
     __A	:       Optional[Any]      =							parser.parse_args()
     if args.model_size != "tokenizer_only":
          write_model(
              model_path=args.output_dir       ,					input_base_path=os.path.join(args.input_dir       ,					args.model_size							)       ,					model_size=args.model_size       ,					safe_serialization=args.safe_serialization       ,					)
     __A	:       List[Any]      =							os.path.join(args.input_dir       ,					"tokenizer.model"							)
     write_tokenizer(args.output_dir       ,					SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
      main()





'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
      import PIL


_UpperCamelCase 							=  logging.get_logger(__name__)


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       int      =							b.T
     __A	:       Optional[Any]      =							np.sum(np.square(SCREAMING_SNAKE_CASE							)       ,					axis=1							)
     __A	:       List[Any]      =							np.sum(np.square(SCREAMING_SNAKE_CASE							)       ,					axis=0							)
     __A	:       List[Any]      =							np.matmul(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
     __A	:       str      =							aa[:, None] - 2 * ab + ba[None, :]
     return d


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       Union[str, Any]      =							x.reshape(-1       ,					3							)
     __A	:       Any      =							squared_euclidean_distance(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
     return np.argmin(SCREAMING_SNAKE_CASE       ,					axis=1							)







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				['pixel_values']


     def __init__(   self			,     lowerCamelCase = None			,     lowerCamelCase = True			,     lowerCamelCase = None			,     lowerCamelCase = PILImageResampling.BILINEAR			,     lowerCamelCase = True			,     lowerCamelCase = True			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          super().__init__(**lowerCamelCase		)
          __A	:       Optional[int]      =							size if size is not None else {"height": 256, "width": 256}
          __A	:       int      =							get_size_dict(lowerCamelCase		)
          __A	:       str      =							np.array(lowerCamelCase		) if clusters is not None else None
          __A	:       List[Any]      =							do_resize
          __A	:       Dict      =							size
          __A	:       str      =							resample
          __A	:       Optional[Any]      =							do_normalize
          __A	:       Dict      =							do_color_quantize


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase			,     lowerCamelCase = PILImageResampling.BILINEAR			,     lowerCamelCase = None			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          __A	:       Optional[Any]      =							get_size_dict(lowerCamelCase		)
          if "height" not in size or "width" not in size:
               raise ValueError(f"Size dictionary must contain both height and width keys. Got {size.keys()}"		)
          return resize(
              lowerCamelCase			,     size=(size["height"], size["width"])			,     resample=lowerCamelCase			,     data_format=lowerCamelCase			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None			,     ):


          '''simple docstring'''





          __A	:       List[str]      =							rescale(image=lowerCamelCase			,     scale=1 / 127.5			,     data_format=lowerCamelCase		)
          __A	:       List[Any]      =							image - 1
          return image


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = None			,     lowerCamelCase = ChannelDimension.FIRST			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          __A	:       List[Any]      =							do_resize if do_resize is not None else self.do_resize
          __A	:       int      =							size if size is not None else self.size
          __A	:       Tuple      =							get_size_dict(lowerCamelCase		)
          __A	:       Tuple      =							resample if resample is not None else self.resample
          __A	:       Union[str, Any]      =							do_normalize if do_normalize is not None else self.do_normalize
          __A	:       Tuple      =							do_color_quantize if do_color_quantize is not None else self.do_color_quantize
          __A	:       Any      =							clusters if clusters is not None else self.clusters
          __A	:       Dict      =							np.array(lowerCamelCase		)

          __A	:       Optional[int]      =							make_list_of_images(lowerCamelCase		)

          if not valid_images(lowerCamelCase		):
               raise ValueError(
                   "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                   "torch.Tensor, tf.Tensor or jax.ndarray."		)

          if do_resize and size is None or resample is None:
               raise ValueError("Size and resample must be specified if do_resize is True."		)

          if do_color_quantize and clusters is None:
               raise ValueError("Clusters must be specified if do_color_quantize is True."		)

          # All transformations expect numpy arrays.
          __A	:       Optional[Any]      =							[to_numpy_array(lowerCamelCase		) for image in images]

          if do_resize:
               __A	:       Dict      =							[self.resize(image=lowerCamelCase			,     size=lowerCamelCase			,     resample=lowerCamelCase		) for image in images]

          if do_normalize:
               __A	:       List[Any]      =							[self.normalize(image=lowerCamelCase		) for image in images]

          if do_color_quantize:
               __A	:       Union[str, Any]      =							[to_channel_dimension_format(lowerCamelCase			,     ChannelDimension.LAST		) for image in images]
               # color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
               __A	:       int      =							np.array(lowerCamelCase		)
               __A	:       Any      =							color_quantize(lowerCamelCase			,     lowerCamelCase		).reshape(images.shape[:-1]		)

               # flatten to (batch_size, height*width)
               __A	:       Tuple      =							images.shape[0]
               __A	:       List[Any]      =							images.reshape(lowerCamelCase			,     -1		)

               # We need to convert back to a list of images to keep consistent behaviour across processors.
               __A	:       Optional[int]      =							list(lowerCamelCase		)
          else:
               __A	:       Tuple      =							[to_channel_dimension_format(lowerCamelCase			,     lowerCamelCase		) for image in images]

          __A	:       Optional[int]      =							{"input_ids": images}
          return BatchFeature(data=lowerCamelCase			,     tensor_type=lowerCamelCase		)

