
"""simple docstring"""


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
						)
						A_    							=					None
A_    							=					{
    """7B""": 1_1008,
    """13B""": 1_3824,
    """30B""": 1_7920,
    """65B""": 2_2016,
    """70B""": 2_8672,
}
A_    							=					{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__=1	,lowerCAmelCase__=256						):
			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)






def  lowercase						(	lowerCAmelCase__						):
			with open(lowerCAmelCase__	,'''r'''						) as f:
						return json.load(lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__						):
			with open(lowerCAmelCase__	,'''w'''						) as f:
						json.dump(lowerCAmelCase__	,lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__=True						):
			os.makedirs(lowerCAmelCase__	,exist_ok=lowerCAmelCase__						)
			lowerCamelCase_					=  os.path.join(lowerCAmelCase__	,'''tmp'''						)
			os.makedirs(lowerCAmelCase__	,exist_ok=lowerCAmelCase__						)

			lowerCamelCase_					=  read_json(os.path.join(lowerCAmelCase__	,'''params.json'''						)						)
			lowerCamelCase_					=  NUM_SHARDS[model_size]
			lowerCamelCase_					=  params['''n_layers''']
			lowerCamelCase_					=  params['''n_heads''']
			lowerCamelCase_					=  n_heads // num_shards
			lowerCamelCase_					=  params['''dim''']
			lowerCamelCase_					=  dim // n_heads
			lowerCamelCase_					=  10_000.0
			lowerCamelCase_					=  1.0 / (base ** (torch.arange(0	,lowerCAmelCase__	,2						).float() / dims_per_head))

			if "n_kv_heads" in params:
						lowerCamelCase_					=  params['''n_kv_heads''']  # for GQA / MQA
						lowerCamelCase_					=  n_heads_per_shard // num_key_value_heads
						lowerCamelCase_					=  dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						lowerCamelCase_					=  n_heads
						lowerCamelCase_					=  n_heads_per_shard
						lowerCamelCase_					=  dim

			# permute for sliced rotary
			def permute(lowerCAmelCase__	,lowerCAmelCase__=n_heads	,lowerCAmelCase__=dim	,lowerCAmelCase__=dim						):
						return w.view(lowerCAmelCase__	,dima // n_heads // 2	,2	,lowerCAmelCase__						).transpose(1	,2						).reshape(lowerCAmelCase__	,lowerCAmelCase__						)

			print(f"Fetching all parameters from the checkpoint at {input_base_path}."						)
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						lowerCamelCase_					=  torch.load(os.path.join(lowerCAmelCase__	,'''consolidated.00.pth'''						)	,map_location='''cpu'''						)
			else:
						# Sharded
						lowerCamelCase_					=  [
						    torch.load(os.path.join(lowerCAmelCase__	,f"consolidated.{i:02d}.pth"						)	,map_location='''cpu'''						)
						    for i in range(lowerCAmelCase__						)
						]
			lowerCamelCase_					=  0
			lowerCamelCase_					=  {'''weight_map''': {}}
			for layer_i in range(lowerCAmelCase__						):
						lowerCamelCase_					=  f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
						if model_size == "7B":
									# Unsharded
									lowerCamelCase_					=  {
									    f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
									        loaded[f"layers.{layer_i}.attention.wq.weight"]						),
									    f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
									        loaded[f"layers.{layer_i}.attention.wk.weight"]						),
									    f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
									    f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
									    f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
									    f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
									    f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
									    f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
									    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									lowerCamelCase_					=  {
									    f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
									        f"layers.{layer_i}.attention_norm.weight"
									    ].clone(),
									    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
									        f"layers.{layer_i}.ffn_norm.weight"
									    ].clone(),
									}
									lowerCamelCase_					=  permute(
									    torch.cat(
									        [
									            loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									            for i in range(lowerCAmelCase__						)
									        ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)						)
									lowerCamelCase_					=  permute(
									    torch.cat(
									        [
									            loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
									                lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									            for i in range(lowerCAmelCase__						)
									        ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)	,lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__	,)
									lowerCamelCase_					=  torch.cat(
									    [
									        loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
									            lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									        for i in range(lowerCAmelCase__						)
									    ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)

									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(lowerCAmelCase__						)]	,dim=1						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(lowerCAmelCase__						)]	,dim=0						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(lowerCAmelCase__						)]	,dim=1						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(lowerCAmelCase__						)]	,dim=0						)

						lowerCamelCase_					=  inv_freq
						for k, v in state_dict.items():
									lowerCamelCase_					=  filename
									param_count += v.numel()
						torch.save(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,lowerCAmelCase__						)						)

			lowerCamelCase_					=  f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
			if model_size == "7B":
						# Unsharded
						lowerCamelCase_					=  {
						    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
						    '''model.norm.weight''': loaded['''norm.weight'''],
						    '''lm_head.weight''': loaded['''output.weight'''],
						}
			else:
						lowerCamelCase_					=  {
						    '''model.norm.weight''': loaded[0]['''norm.weight'''],
						    '''model.embed_tokens.weight''': torch.cat(
						        [loaded[i]['''tok_embeddings.weight'''] for i in range(lowerCAmelCase__						)]	,dim=1						),
						    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(lowerCAmelCase__						)]	,dim=0						),
						}

			for k, v in state_dict.items():
						lowerCamelCase_					=  filename
						param_count += v.numel()
			torch.save(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,lowerCAmelCase__						)						)

			# Write configs
			lowerCamelCase_					=  {'''total_size''': param_count * 2}
			write_json(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,'''pytorch_model.bin.index.json'''						)						)
			lowerCamelCase_					=  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
			lowerCamelCase_					=  params['''multiple_of'''] if '''multiple_of''' in params else 256
			lowerCamelCase_					=  LlamaConfig(
			    hidden_size=lowerCAmelCase__	,intermediate_size=compute_intermediate_size(lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)	,num_attention_heads=params['''n_heads''']	,num_hidden_layers=params['''n_layers''']	,rms_norm_eps=params['''norm_eps''']	,num_key_value_heads=lowerCAmelCase__	,)
			config.save_pretrained(lowerCAmelCase__						)

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print('''Loading the checkpoint in a Llama model.'''						)
			lowerCamelCase_					=  LlamaForCausalLM.from_pretrained(lowerCAmelCase__	,torch_dtype=torch.floataa	,low_cpu_mem_usage=lowerCAmelCase__						)
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print('''Saving in the Transformers format.'''						)
			model.save_pretrained(lowerCAmelCase__	,safe_serialization=lowerCAmelCase__						)
			shutil.rmtree(lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__						):
			# Initialize the tokenizer based on the `spm` model
			lowerCamelCase_					=  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."						)
			lowerCamelCase_					=  tokenizer_class(lowerCAmelCase__						)
			tokenizer.save_pretrained(lowerCAmelCase__						)






def  lowercase						(	):
			lowerCamelCase_					=  argparse.ArgumentParser()
			parser.add_argument(
			    '''--input_dir'''	,help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''	,)
			parser.add_argument(
			    '''--model_size'''	,choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']	,)
			parser.add_argument(
			    '''--output_dir'''	,help='''Location to write HF model and tokenizer'''	,)
			parser.add_argument('''--safe_serialization'''	,type=lowerCAmelCase__	,help='''Whether or not to save using `safetensors`.'''						)
			lowerCamelCase_					=  parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir	,input_base_path=os.path.join(args.input_dir	,args.model_size						)	,model_size=args.model_size	,safe_serialization=args.safe_serialization	,)
			lowerCamelCase_					=  os.path.join(args.input_dir	,'''tokenizer.model'''						)
			write_tokenizer(args.output_dir	,lowerCAmelCase__						)


if __name__ == "__main__":
						main()





"""simple docstring"""


import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling






def  lowercase						(	lowerCAmelCase__						):
			lowerCamelCase_					=  [2, 2, 6, 2] if '''tiny''' in model_name else [2, 2, 18, 2]
			lowerCamelCase_					=  True if '''large''' in model_name or '''huge''' in model_name else False
			lowerCamelCase_					=  True if '''large''' in model_name or '''huge''' in model_name else False
			lowerCamelCase_					=  True if '''large''' in model_name or '''huge''' in model_name else False

			if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
						if "fl3" in model_name:
									lowerCamelCase_					=  [3, 3, 3, 3]
									lowerCamelCase_					=  [5, 5, 5, 5]
						elif "fl4" in model_name:
									lowerCamelCase_					=  [4, 4, 4, 4]
									lowerCamelCase_					=  [3, 3, 3, 3]

			if "tiny" in model_name or "small" in model_name or "base" in model_name:
						lowerCamelCase_					=  [3, 3, 3, 3]
						if "lrf" in model_name:
									lowerCamelCase_					=  [3, 3, 3, 3]
						else:
									lowerCamelCase_					=  [2, 2, 2, 2]

			if "tiny" in model_name:
						lowerCamelCase_					=  96
			elif "small" in model_name:
						lowerCamelCase_					=  96
			elif "base" in model_name:
						lowerCamelCase_					=  128
			elif "large" in model_name:
						lowerCamelCase_					=  192
			elif "xlarge" in model_name:
						lowerCamelCase_					=  256
			elif "huge" in model_name:
						lowerCamelCase_					=  352

			# set label information
			lowerCamelCase_					=  '''huggingface/label-files'''
			if "large" in model_name or "huge" in model_name:
						lowerCamelCase_					=  '''imagenet-22k-id2label.json'''
			else:
						lowerCamelCase_					=  '''imagenet-1k-id2label.json'''

			lowerCamelCase_					=  json.load(open(hf_hub_download(lowerCAmelCase__	,lowerCAmelCase__	,repo_type='''dataset'''						)	,'''r'''						)						)
			lowerCamelCase_					=  {int(lowerCAmelCase__						): v for k, v in idalabel.items()}
			lowerCamelCase_					=  {v: k for k, v in idalabel.items()}

			lowerCamelCase_					=  FocalNetConfig(
			    embed_dim=lowerCAmelCase__	,depths=lowerCAmelCase__	,focal_levels=lowerCAmelCase__	,focal_windows=lowerCAmelCase__	,use_conv_embed=lowerCAmelCase__	,idalabel=lowerCAmelCase__	,labelaid=lowerCAmelCase__	,use_post_layernorm=lowerCAmelCase__	,use_layerscale=lowerCAmelCase__	,)

			return config






def  lowercase						(	lowerCAmelCase__						):
			if "patch_embed.proj" in name:
						lowerCamelCase_					=  name.replace('''patch_embed.proj'''	,'''embeddings.patch_embeddings.projection'''						)
			if "patch_embed.norm" in name:
						lowerCamelCase_					=  name.replace('''patch_embed.norm'''	,'''embeddings.norm'''						)
			if "layers" in name:
						lowerCamelCase_					=  '''encoder.''' + name
			if "encoder.layers" in name:
						lowerCamelCase_					=  name.replace('''encoder.layers'''	,'''encoder.stages'''						)
			if "downsample.proj" in name:
						lowerCamelCase_					=  name.replace('''downsample.proj'''	,'''downsample.projection'''						)
			if "blocks" in name:
						lowerCamelCase_					=  name.replace('''blocks'''	,'''layers'''						)
			if "modulation.f.weight" in name or "modulation.f.bias" in name:
						lowerCamelCase_					=  name.replace('''modulation.f'''	,'''modulation.projection_in'''						)
			if "modulation.h.weight" in name or "modulation.h.bias" in name:
						lowerCamelCase_					=  name.replace('''modulation.h'''	,'''modulation.projection_context'''						)
			if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
						lowerCamelCase_					=  name.replace('''modulation.proj'''	,'''modulation.projection_out'''						)

			if name == "norm.weight":
						lowerCamelCase_					=  '''layernorm.weight'''
			if name == "norm.bias":
						lowerCamelCase_					=  '''layernorm.bias'''

			if "head" in name:
						lowerCamelCase_					=  name.replace('''head'''	,'''classifier'''						)
			else:
						lowerCamelCase_					=  '''focalnet.''' + name

			return name






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__=False						):
			# fmt: off
			lowerCamelCase_					=  {
			    '''focalnet-tiny''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth''',
			    '''focalnet-tiny-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth''',
			    '''focalnet-small''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth''',
			    '''focalnet-small-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth''',
			    '''focalnet-base''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth''',
			    '''focalnet-base-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth''',
			    '''focalnet-large-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth''',
			    '''focalnet-large-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth''',
			    '''focalnet-xlarge-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth''',
			    '''focalnet-xlarge-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth''',
			}
			# fmt: on

			lowerCamelCase_					=  model_name_to_url[model_name]
			print('''Checkpoint URL: '''	,lowerCAmelCase__						)
			lowerCamelCase_					=  torch.hub.load_state_dict_from_url(lowerCAmelCase__	,map_location='''cpu'''						)['''model''']

			# rename keys
			for key in state_dict.copy().keys():
						lowerCamelCase_					=  state_dict.pop(lowerCAmelCase__						)
						lowerCamelCase_					=  val

			lowerCamelCase_					=  get_focalnet_config(lowerCAmelCase__						)
			lowerCamelCase_					=  FocalNetForImageClassification(lowerCAmelCase__						)
			model.eval()

			# load state dict
			model.load_state_dict(lowerCAmelCase__						)

			# verify conversion
			lowerCamelCase_					=  '''http://images.cocodataset.org/val2017/000000039769.jpg'''

			lowerCamelCase_					=  BitImageProcessor(
			    do_resize=lowerCAmelCase__	,size={'''shortest_edge''': 256}	,resample=PILImageResampling.BILINEAR	,do_center_crop=lowerCAmelCase__	,crop_size=224	,do_normalize=lowerCAmelCase__	,image_mean=lowerCAmelCase__	,image_std=lowerCAmelCase__	,)
			lowerCamelCase_					=  Image.open(requests.get(lowerCAmelCase__	,stream=lowerCAmelCase__						).raw						)
			lowerCamelCase_					=  processor(images=lowerCAmelCase__	,return_tensors='''pt'''						)

			lowerCamelCase_					=  transforms.Compose(
			    [
			        transforms.Resize(256						),
			        transforms.CenterCrop(224						),
			        transforms.ToTensor(),
			        transforms.Normalize(mean=[0.485, 0.456, 0.406]	,std=[0.229, 0.224, 0.225]						),
			    ]						)

			lowerCamelCase_					=  image_transforms(lowerCAmelCase__						).unsqueeze(0						)

			# verify pixel_values
			assert torch.allclose(inputs.pixel_values	,lowerCAmelCase__	,atol=1E-4						)

			lowerCamelCase_					=  model(**lowerCAmelCase__						)

			lowerCamelCase_					=  outputs.logits.argmax(-1						).item()
			print('''Predicted class:'''	,model.config.idalabel[predicted_class_idx]						)

			print('''First values of logits:'''	,outputs.logits[0, :3]						)

			if model_name == "focalnet-tiny":
						lowerCamelCase_					=  torch.tensor([0.2_166, -0.4_368, 0.2_191]						)
			elif model_name == "focalnet-tiny-lrf":
						lowerCamelCase_					=  torch.tensor([1.1_669, 0.0_125, -0.1_695]						)
			elif model_name == "focalnet-small":
						lowerCamelCase_					=  torch.tensor([0.4_917, -0.0_430, 0.1_341]						)
			elif model_name == "focalnet-small-lrf":
						lowerCamelCase_					=  torch.tensor([-0.2_588, -0.5_342, -0.2_331]						)
			elif model_name == "focalnet-base":
						lowerCamelCase_					=  torch.tensor([-0.1_655, -0.4_090, -0.1_730]						)
			elif model_name == "focalnet-base-lrf":
						lowerCamelCase_					=  torch.tensor([0.5_306, -0.0_483, -0.3_928]						)
			assert torch.allclose(outputs.logits[0, :3]	,lowerCAmelCase__	,atol=1E-4						)
			print('''Looks ok!'''						)

			if pytorch_dump_folder_path is not None:
						print(f"Saving model and processor of {model_name} to {pytorch_dump_folder_path}"						)
						model.save_pretrained(lowerCAmelCase__						)
						processor.save_pretrained(lowerCAmelCase__						)

			if push_to_hub:
						print(f"Pushing model and processor of {model_name} to the hub..."						)
						model.push_to_hub(f"{model_name}"						)
						processor.push_to_hub(f"{model_name}"						)


if __name__ == "__main__":
						A_    							=					argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--model_name""",
						    default="""focalnet-tiny""",
						    type=str,
						    help="""Name of the FocalNet model you'd like to convert.""",
						)
						parser.add_argument(
						    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
						)
						parser.add_argument(
						    """--push_to_hub""",
						    action="""store_true""",
						    help="""Whether to push the model and processor to the hub.""",
						)

						A_    							=					parser.parse_args()
						convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)



