
'''simple docstring'''
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
    from transformers import LlamaTokenizerFast
except ImportError as e:
    warnings.warn(e)
    warnings.warn(
        'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
    )
    __lowerCAmelCase												=	None
__lowerCAmelCase												=	{
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
__lowerCAmelCase												=	{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE=1							, _SCREAMING_SNAKE_CASE=256				):
       return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3				)				) + multiple_of - 1) // multiple_of)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):
       with open(lowerCAmelCase_							, """r"""				) as f:
              return json.load(lowerCAmelCase_				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
       with open(lowerCAmelCase_							, """w"""				) as f:
              json.dump(lowerCAmelCase_							, lowerCAmelCase_				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE=True				):
       os.makedirs(lowerCAmelCase_							, exist_ok=lowerCAmelCase_				)
       _snake_case       =  os.path.join(lowerCAmelCase_							, """tmp"""				)
       os.makedirs(lowerCAmelCase_							, exist_ok=lowerCAmelCase_				)

       _snake_case       =  read_json(os.path.join(lowerCAmelCase_							, """params.json"""				)				)
       _snake_case       =  NUM_SHARDS[model_size]
       _snake_case       =  params['n_layers']
       _snake_case       =  params['n_heads']
       _snake_case       =  n_heads // num_shards
       _snake_case       =  params['dim']
       _snake_case       =  dim // n_heads
       _snake_case       =  1_0000.0
       _snake_case       =  1.0 / (base ** (torch.arange(0							, lowerCAmelCase_							, 2				).float() / dims_per_head))

       if "n_kv_heads" in params:
              _snake_case       =  params['n_kv_heads']  # for GQA / MQA
              _snake_case       =  n_heads_per_shard // num_key_value_heads
              _snake_case       =  dim // num_key_value_heads
       else:  # compatibility with other checkpoints
              _snake_case       =  n_heads
              _snake_case       =  n_heads_per_shard
              _snake_case       =  dim

       # permute for sliced rotary
       def permute(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE=n_heads							, _SCREAMING_SNAKE_CASE=dim							, _SCREAMING_SNAKE_CASE=dim				):
              return w.view(lowerCAmelCase_							, dima // n_heads // 2							, 2							, lowerCAmelCase_				).transpose(1							, 2				).reshape(lowerCAmelCase_							, lowerCAmelCase_				)

       print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""				)
       # Load weights
       if model_size == "7B":
              # Not sharded
              # (The sharded implementation would also work, but this is simpler.)
              _snake_case       =  torch.load(os.path.join(lowerCAmelCase_							, """consolidated.00.pth"""				)							, map_location="""cpu"""				)
       else:
              # Sharded
              _snake_case       =  [
                  torch.load(os.path.join(lowerCAmelCase_							, f"""consolidated.{i:02d}.pth"""				)							, map_location="""cpu"""				)
                  for i in range(lowerCAmelCase_				)
              ]
       _snake_case       =  0
       _snake_case       =  {'weight_map': {}}
       for layer_i in range(lowerCAmelCase_				):
              _snake_case       =  f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
              if model_size == "7B":
                     # Unsharded
                     _snake_case       =  {
                         f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                             loaded[f"""layers.{layer_i}.attention.wq.weight"""]				),
                         f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                             loaded[f"""layers.{layer_i}.attention.wk.weight"""]				),
                         f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
                         f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
                         f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
                         f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
                         f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
                         f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
                         f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
                     }
              else:
                     # Sharded
                     # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                     # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                     # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                     _snake_case       =  {
                         f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                             f"""layers.{layer_i}.attention_norm.weight"""
                         ].clone(),
                         f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                             f"""layers.{layer_i}.ffn_norm.weight"""
                         ].clone(),
                     }
                     _snake_case       =  permute(
                         torch.cat(
                             [
                                 loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_				)
                                 for i in range(lowerCAmelCase_				)
                             ]							, dim=0							, ).reshape(lowerCAmelCase_							, lowerCAmelCase_				)				)
                     _snake_case       =  permute(
                         torch.cat(
                             [
                                 loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                                     lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_				)
                                 for i in range(lowerCAmelCase_				)
                             ]							, dim=0							, ).reshape(lowerCAmelCase_							, lowerCAmelCase_				)							, lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_							, )
                     _snake_case       =  torch.cat(
                         [
                             loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                                 lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_				)
                             for i in range(lowerCAmelCase_				)
                         ]							, dim=0							, ).reshape(lowerCAmelCase_							, lowerCAmelCase_				)

                     _snake_case       =  torch.cat(
                         [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(lowerCAmelCase_				)]							, dim=1				)
                     _snake_case       =  torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(lowerCAmelCase_				)]							, dim=0				)
                     _snake_case       =  torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(lowerCAmelCase_				)]							, dim=1				)
                     _snake_case       =  torch.cat(
                         [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(lowerCAmelCase_				)]							, dim=0				)

              _snake_case       =  inv_freq
              for k, v in state_dict.items():
                     _snake_case       =  filename
                     param_count += v.numel()
              torch.save(lowerCAmelCase_							, os.path.join(lowerCAmelCase_							, lowerCAmelCase_				)				)

       _snake_case       =  f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
       if model_size == "7B":
              # Unsharded
              _snake_case       =  {
                  'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
                  'model.norm.weight': loaded['norm.weight'],
                  'lm_head.weight': loaded['output.weight'],
              }
       else:
              _snake_case       =  {
                  'model.norm.weight': loaded[0]['norm.weight'],
                  'model.embed_tokens.weight': torch.cat(
                      [loaded[i]["""tok_embeddings.weight"""] for i in range(lowerCAmelCase_				)]							, dim=1				),
                  'lm_head.weight': torch.cat([loaded[i]["""output.weight"""] for i in range(lowerCAmelCase_				)]							, dim=0				),
              }

       for k, v in state_dict.items():
              _snake_case       =  filename
              param_count += v.numel()
       torch.save(lowerCAmelCase_							, os.path.join(lowerCAmelCase_							, lowerCAmelCase_				)				)

       # Write configs
       _snake_case       =  {'total_size': param_count * 2}
       write_json(lowerCAmelCase_							, os.path.join(lowerCAmelCase_							, """pytorch_model.bin.index.json"""				)				)
       _snake_case       =  params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
       _snake_case       =  params['multiple_of'] if 'multiple_of' in params else 256
       _snake_case       =  LlamaConfig(
           hidden_size=lowerCAmelCase_							, intermediate_size=compute_intermediate_size(lowerCAmelCase_							, lowerCAmelCase_							, lowerCAmelCase_				)							, num_attention_heads=params["""n_heads"""]							, num_hidden_layers=params["""n_layers"""]							, rms_norm_eps=params["""norm_eps"""]							, num_key_value_heads=lowerCAmelCase_							, )
       config.save_pretrained(lowerCAmelCase_				)

       # Make space so we can load the model properly now.
       del state_dict
       del loaded
       gc.collect()

       print("""Loading the checkpoint in a Llama model."""				)
       _snake_case       =  LlamaForCausalLM.from_pretrained(lowerCAmelCase_							, torch_dtype=torch.floataa							, low_cpu_mem_usage=lowerCAmelCase_				)
       # Avoid saving this as part of the config.
       del model.config._name_or_path

       print("""Saving in the Transformers format."""				)
       model.save_pretrained(lowerCAmelCase_							, safe_serialization=lowerCAmelCase_				)
       shutil.rmtree(lowerCAmelCase_				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
       # Initialize the tokenizer based on the `spm` model
       _snake_case       =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
       print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""				)
       _snake_case       =  tokenizer_class(lowerCAmelCase_				)
       tokenizer.save_pretrained(lowerCAmelCase_				)






def  __SCREAMING_SNAKE_CASE				(  ):
       _snake_case       =  argparse.ArgumentParser()
       parser.add_argument(
           """--input_dir"""							, help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""							, )
       parser.add_argument(
           """--model_size"""							, choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]							, )
       parser.add_argument(
           """--output_dir"""							, help="""Location to write HF model and tokenizer"""							, )
       parser.add_argument("""--safe_serialization"""							, type=lowerCAmelCase_							, help="""Whether or not to save using `safetensors`."""				)
       _snake_case       =  parser.parse_args()
       if args.model_size != "tokenizer_only":
              write_model(
                  model_path=args.output_dir							, input_base_path=os.path.join(args.input_dir							, args.model_size				)							, model_size=args.model_size							, safe_serialization=args.safe_serialization							, )
       _snake_case       =  os.path.join(args.input_dir							, """tokenizer.model"""				)
       write_tokenizer(args.output_dir							, lowerCAmelCase_				)


if __name__ == "__main__":
    main()

'''simple docstring'''
import gc
import unittest

from diffusers import FlaxControlNetModel, FlaxStableDiffusionControlNetPipeline
from diffusers.utils import is_flax_available, load_image, slow
from diffusers.utils.testing_utils import require_flax


if is_flax_available():
				import jax
				import jax.numpy as jnp
				from flax.jax_utils import replicate
				from flax.training.common_utils import shard


@slow
@require_flax
class       _lowerCAmelCase   (   unittest.TestCase				):







							'''simple docstring'''

							def 	lowercase							(self  )					->       Optional[int]:
														# clean up the VRAM after each test
														super().tearDown()
														gc.collect()

							def 	lowercase							(self  )					->       Dict:
														_snake_case,							_snake_case       =  FlaxControlNetModel.from_pretrained(
														    """lllyasviel/sd-controlnet-canny"""  ,  from_pt=UpperCAmelCase  ,  dtype=jnp.bfloataa  )
														_snake_case,							_snake_case       =  FlaxStableDiffusionControlNetPipeline.from_pretrained(
														    """runwayml/stable-diffusion-v1-5"""  ,  controlnet=UpperCAmelCase  ,  from_pt=UpperCAmelCase  ,  dtype=jnp.bfloataa  )
														_snake_case       =  controlnet_params

														_snake_case       =  """bird"""
														_snake_case       =  jax.device_count()
														_snake_case       =  pipe.prepare_text_inputs([prompts] * num_samples  )

														_snake_case       =  load_image(
														    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"""  )
														_snake_case       =  pipe.prepare_image_inputs([canny_image] * num_samples  )

														_snake_case       =  jax.random.PRNGKey(0  )
														_snake_case       =  jax.random.split(UpperCAmelCase  ,  jax.device_count()  )

														_snake_case       =  replicate(UpperCAmelCase  )
														_snake_case       =  shard(UpperCAmelCase  )
														_snake_case       =  shard(UpperCAmelCase  )

														_snake_case       =  pipe(
														    prompt_ids=UpperCAmelCase  ,  image=UpperCAmelCase  ,  params=UpperCAmelCase  ,  prng_seed=UpperCAmelCase  ,  num_inference_steps=50  ,  jit=UpperCAmelCase  ,  ).images
														assert images.shape == (jax.device_count(), 1, 768, 512, 3)

														_snake_case       =  images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]  )
														_snake_case       =  images[0, 253:256, 253:256, -1]

														_snake_case       =  jnp.asarray(jax.device_get(image_slice.flatten()  )  )
														_snake_case       =  jnp.array(
														    [0.16_7969, 0.11_6699, 0.08_1543, 0.15_4297, 0.13_2812, 0.10_8887, 0.16_9922, 0.16_9922, 0.20_5078]  )
														print(f"""output_slice: {output_slice}"""  )
														assert jnp.abs(output_slice - expected_slice  ).max() < 1e-2





							def 	lowercase							(self  )					->       Optional[int]:
														_snake_case,							_snake_case       =  FlaxControlNetModel.from_pretrained(
														    """lllyasviel/sd-controlnet-openpose"""  ,  from_pt=UpperCAmelCase  ,  dtype=jnp.bfloataa  )
														_snake_case,							_snake_case       =  FlaxStableDiffusionControlNetPipeline.from_pretrained(
														    """runwayml/stable-diffusion-v1-5"""  ,  controlnet=UpperCAmelCase  ,  from_pt=UpperCAmelCase  ,  dtype=jnp.bfloataa  )
														_snake_case       =  controlnet_params

														_snake_case       =  """Chef in the kitchen"""
														_snake_case       =  jax.device_count()
														_snake_case       =  pipe.prepare_text_inputs([prompts] * num_samples  )

														_snake_case       =  load_image(
														    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose.png"""  )
														_snake_case       =  pipe.prepare_image_inputs([pose_image] * num_samples  )

														_snake_case       =  jax.random.PRNGKey(0  )
														_snake_case       =  jax.random.split(UpperCAmelCase  ,  jax.device_count()  )

														_snake_case       =  replicate(UpperCAmelCase  )
														_snake_case       =  shard(UpperCAmelCase  )
														_snake_case       =  shard(UpperCAmelCase  )

														_snake_case       =  pipe(
														    prompt_ids=UpperCAmelCase  ,  image=UpperCAmelCase  ,  params=UpperCAmelCase  ,  prng_seed=UpperCAmelCase  ,  num_inference_steps=50  ,  jit=UpperCAmelCase  ,  ).images
														assert images.shape == (jax.device_count(), 1, 768, 512, 3)

														_snake_case       =  images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]  )
														_snake_case       =  images[0, 253:256, 253:256, -1]

														_snake_case       =  jnp.asarray(jax.device_get(image_slice.flatten()  )  )
														_snake_case       =  jnp.array(
														    [[0.27_1484, 0.26_1719, 0.27_5391, 0.27_7344, 0.27_9297, 0.29_1016, 0.29_4922, 0.30_2734, 0.30_2734]]  )
														print(f"""output_slice: {output_slice}"""  )
														assert jnp.abs(output_slice - expected_slice  ).max() < 1e-2