


'''simple docstring'''



import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							List[Any] ,       UpperCAmelCase_       :							Tuple=False				)	->						Union[str, Any]:
     __lowerCamelCase  :      Union[str, Any]														=						[]
     for i in range(config.num_hidden_layers				):
          # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
          rename_keys.append((F'module.blocks.{i}.norm1.weight', F'vit.encoder.layer.{i}.layernorm_before.weight')				)
          rename_keys.append((F'module.blocks.{i}.norm1.bias', F'vit.encoder.layer.{i}.layernorm_before.bias')				)
          rename_keys.append(
              (F'module.blocks.{i}.attn.proj.weight', F'vit.encoder.layer.{i}.attention.output.dense.weight')				)
          rename_keys.append((F'module.blocks.{i}.attn.proj.bias', F'vit.encoder.layer.{i}.attention.output.dense.bias')				)
          rename_keys.append((F'module.blocks.{i}.norm2.weight', F'vit.encoder.layer.{i}.layernorm_after.weight')				)
          rename_keys.append((F'module.blocks.{i}.norm2.bias', F'vit.encoder.layer.{i}.layernorm_after.bias')				)
          rename_keys.append((F'module.blocks.{i}.mlp.fc1.weight', F'vit.encoder.layer.{i}.intermediate.dense.weight')				)
          rename_keys.append((F'module.blocks.{i}.mlp.fc1.bias', F'vit.encoder.layer.{i}.intermediate.dense.bias')				)
          rename_keys.append((F'module.blocks.{i}.mlp.fc2.weight', F'vit.encoder.layer.{i}.output.dense.weight')				)
          rename_keys.append((F'module.blocks.{i}.mlp.fc2.bias', F'vit.encoder.layer.{i}.output.dense.bias')				)

     # projection layer + position embeddings
     rename_keys.extend(
         [
             ('module.cls_token', 'vit.embeddings.cls_token'),
             ('module.patch_embed.proj.weight', 'vit.embeddings.patch_embeddings.projection.weight'),
             ('module.patch_embed.proj.bias', 'vit.embeddings.patch_embeddings.projection.bias'),
             ('module.pos_embed', 'vit.embeddings.position_embeddings'),
         ]				)

     if base_model:
          # layernorm + pooler
          rename_keys.extend(
              [
                  ('module.norm.weight', 'layernorm.weight'),
                  ('module.norm.bias', 'layernorm.bias'),
              ]				)

          # if just the base model, we should remove "vit" from all keys that start with "vit"
          __lowerCamelCase  :      Optional[Any]														=						[(pair[0], pair[1][4:]) if pair[1].startswith('vit'				) else pair for pair in rename_keys]
     else:
          # layernorm + classification head
          rename_keys.extend(
              [
                  ('norm.weight', 'vit.layernorm.weight'),
                  ('norm.bias', 'vit.layernorm.bias'),
                  ('head.weight', 'classifier.weight'),
                  ('head.bias', 'classifier.bias'),
              ]				)

     return rename_keys



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Dict ,       UpperCAmelCase_       :							str ,       UpperCAmelCase_       :							Optional[Any]=False				)	->						Optional[int]:
     for i in range(config.num_hidden_layers				):
          if base_model:
               __lowerCamelCase  :      List[str]														=						''
          else:
               __lowerCamelCase  :      List[Any]														=						'vit.'
          # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
          __lowerCamelCase  :      Optional[Any]														=						state_dict.pop(F'module.blocks.{i}.attn.qkv.weight'				)
          __lowerCamelCase  :      Optional[Any]														=						state_dict.pop(F'module.blocks.{i}.attn.qkv.bias'				)
          # next, add query, keys and values (in that order) to the state dict
          __lowerCamelCase  :      List[Any]														=						in_proj_weight[
              : config.hidden_size, :
          ]
          __lowerCamelCase  :      Dict														=						in_proj_bias[: config.hidden_size]
          __lowerCamelCase  :      str														=						in_proj_weight[
              config.hidden_size : config.hidden_size * 2, :
          ]
          __lowerCamelCase  :      Any														=						in_proj_bias[
              config.hidden_size : config.hidden_size * 2
          ]
          __lowerCamelCase  :      List[str]														=						in_proj_weight[
              -config.hidden_size :, :
          ]
          __lowerCamelCase  :      Dict														=						in_proj_bias[-config.hidden_size :]



def      UpperCAmelCase__   (      UpperCAmelCase_       :							List[Any]				)	->						List[str]:
     __lowerCamelCase  :      Any														=						['head.weight', 'head.bias']
     for k in ignore_keys:
          state_dict.pop(UpperCAmelCase_ ,       UpperCAmelCase_				)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Any				)	->						Tuple:
     # projection head is used in the self-supervised pre-training in MSN,
     # for downstream task it's not needed.
     __lowerCamelCase  :      List[Any]														=						[
         'module.fc.fc1.weight',
         'module.fc.fc1.bias',
         'module.fc.bn1.weight',
         'module.fc.bn1.bias',
         'module.fc.bn1.running_mean',
         'module.fc.bn1.running_var',
         'module.fc.bn1.num_batches_tracked',
         'module.fc.fc2.weight',
         'module.fc.fc2.bias',
         'module.fc.bn2.weight',
         'module.fc.bn2.bias',
         'module.fc.bn2.running_mean',
         'module.fc.bn2.running_var',
         'module.fc.bn2.num_batches_tracked',
         'module.fc.fc3.weight',
         'module.fc.fc3.bias',
     ]
     for k in ignore_keys:
          state_dict.pop(UpperCAmelCase_ ,       UpperCAmelCase_				)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Optional[Any] ,       UpperCAmelCase_       :							List[str] ,       UpperCAmelCase_       :							int				)	->						str:
     __lowerCamelCase  :      Any														=						dct.pop(UpperCAmelCase_				)
     __lowerCamelCase  :      Optional[int]														=						val



def      UpperCAmelCase__   (      UpperCAmelCase_       :							List[Any] ,       UpperCAmelCase_       :							List[str]				)	->						str:
     __lowerCamelCase  :      Dict														=						ViTMSNConfig()
     __lowerCamelCase  :      Dict														=						10_00

     __lowerCamelCase  :      Optional[Any]														=						'datasets/huggingface/label-files'
     __lowerCamelCase  :      Tuple														=						'imagenet-1k-id2label.json'
     __lowerCamelCase  :      List[str]														=						json.load(open(hf_hub_download(UpperCAmelCase_ ,       UpperCAmelCase_				) ,       'r'				)				)
     __lowerCamelCase  :      Tuple														=						{int(UpperCAmelCase_				): v for k, v in idalabel.items()}
     __lowerCamelCase  :      Any														=						idalabel
     __lowerCamelCase  :      str														=						{v: k for k, v in idalabel.items()}

     if "s16" in checkpoint_url:
          __lowerCamelCase  :      Union[str, Any]														=						3_84
          __lowerCamelCase  :      Tuple														=						15_36
          __lowerCamelCase  :      int														=						6
     elif "l16" in checkpoint_url:
          __lowerCamelCase  :      Any														=						10_24
          __lowerCamelCase  :      int														=						40_96
          __lowerCamelCase  :      int														=						24
          __lowerCamelCase  :      Any														=						16
          __lowerCamelCase  :      List[str]														=						0.1
     elif "b4" in checkpoint_url:
          __lowerCamelCase  :      Union[str, Any]														=						4
     elif "l7" in checkpoint_url:
          __lowerCamelCase  :      str														=						7
          __lowerCamelCase  :      Union[str, Any]														=						10_24
          __lowerCamelCase  :      List[Any]														=						40_96
          __lowerCamelCase  :      Union[str, Any]														=						24
          __lowerCamelCase  :      Optional[Any]														=						16
          __lowerCamelCase  :      Optional[Any]														=						0.1

     __lowerCamelCase  :      Optional[Any]														=						ViTMSNModel(UpperCAmelCase_				)

     __lowerCamelCase  :      int														=						torch.hub.load_state_dict_from_url(UpperCAmelCase_ ,       map_location='cpu'				)['target_encoder']

     __lowerCamelCase  :      List[Any]														=						ViTImageProcessor(size=config.image_size				)

     remove_projection_head(UpperCAmelCase_				)
     __lowerCamelCase  :      List[Any]														=						create_rename_keys(UpperCAmelCase_ ,       base_model=UpperCAmelCase_				)

     for src, dest in rename_keys:
          rename_key(UpperCAmelCase_ ,       UpperCAmelCase_ ,       UpperCAmelCase_				)
     read_in_q_k_v(UpperCAmelCase_ ,       UpperCAmelCase_ ,       base_model=UpperCAmelCase_				)

     model.load_state_dict(UpperCAmelCase_				)
     model.eval()

     __lowerCamelCase  :      Any														=						'http://images.cocodataset.org/val2017/000000039769.jpg'

     __lowerCamelCase  :      Optional[Any]														=						Image.open(requests.get(UpperCAmelCase_ ,       stream=UpperCAmelCase_				).raw				)
     __lowerCamelCase  :      str														=						ViTImageProcessor(
         size=config.image_size ,       image_mean=UpperCAmelCase_ ,       image_std=UpperCAmelCase_				)
     __lowerCamelCase  :      Tuple														=						image_processor(images=UpperCAmelCase_ ,       return_tensors='pt'				)

     # forward pass
     torch.manual_seed(2				)
     __lowerCamelCase  :      int														=						model(**UpperCAmelCase_				)
     __lowerCamelCase  :      Optional[int]														=						outputs.last_hidden_state

     # The following Colab Notebook was used to generate these outputs:
     # https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
     if "s16" in checkpoint_url:
          __lowerCamelCase  :      Optional[Any]														=						torch.tensor([[-1.0_915, -1.4_876, -1.1_809]]				)
     elif "b16" in checkpoint_url:
          __lowerCamelCase  :      List[Any]														=						torch.tensor([[14.2_889, -18.9_045, 11.7_281]]				)
     elif "l16" in checkpoint_url:
          __lowerCamelCase  :      str														=						torch.tensor([[41.5_028, -22.8_681, 45.6_475]]				)
     elif "b4" in checkpoint_url:
          __lowerCamelCase  :      int														=						torch.tensor([[-4.3_868, 5.2_932, -0.4_137]]				)
     else:
          __lowerCamelCase  :      Dict														=						torch.tensor([[-0.1_792, -0.6_465, 2.4_263]]				)

     # verify logits
     assert torch.allclose(last_hidden_state[:, 0, :3] ,       UpperCAmelCase_ ,       atol=1e-4				)

     print(F'Saving model to {pytorch_dump_folder_path}'				)
     model.save_pretrained(UpperCAmelCase_				)

     print(F'Saving image processor to {pytorch_dump_folder_path}'				)
     image_processor.save_pretrained(UpperCAmelCase_				)


if __name__ == "__main__":
      A__						:   List[Any]				=    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--checkpoint_url""",
          default="""https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar""",
          type=str,
          help="""URL of the checkpoint you'd like to convert.""",
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
      )

      A__						:   int				=    parser.parse_args()
      convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)










'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__						:   Tuple				=    logging.get_logger(__name__)

A__						:   Dict				=    {
    """RWKV/rwkv-4-169m-pile""": """https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-430m-pile""": """https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-1b5-pile""": """https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-3b-pile""": """https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-7b-pile""": """https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-14b-pile""": """https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json""",
    """RWKV/rwkv-raven-1b5""": """https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json""",
    """RWKV/rwkv-raven-3b""": """https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json""",
    """RWKV/rwkv-raven-7b""": """https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json""",
    """RWKV/rwkv-raven-14b""": """https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json""",
}





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    lowerCamelCase    :   List[Any]					=     'rwkv'
    lowerCamelCase    :   Any					=     {'max_position_embeddings': 'context_length'}







    def __init__(				self     ,	SCREAMING_SNAKE_CASE_=5_02_77     ,	SCREAMING_SNAKE_CASE_=10_24     ,	SCREAMING_SNAKE_CASE_=40_96     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=1E-5     ,	SCREAMING_SNAKE_CASE_=0     ,	SCREAMING_SNAKE_CASE_=0     ,	SCREAMING_SNAKE_CASE_=6     ,	SCREAMING_SNAKE_CASE_=False     ,	SCREAMING_SNAKE_CASE_=True     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Optional[Any]:
         __lowerCamelCase  :      Optional[int]														=						vocab_size
         __lowerCamelCase  :      Tuple														=						context_length
         __lowerCamelCase  :      str														=						hidden_size
         __lowerCamelCase  :      List[str]														=						num_hidden_layers
         __lowerCamelCase  :      Any														=						attention_hidden_size if attention_hidden_size is not None else hidden_size
         __lowerCamelCase  :      Optional[int]														=						intermediate_size if intermediate_size is not None else 4 * hidden_size
         __lowerCamelCase  :      Optional[Any]														=						layer_norm_epsilon
         __lowerCamelCase  :      int														=						rescale_every
         __lowerCamelCase  :      Tuple														=						use_cache

         __lowerCamelCase  :      int														=						bos_token_id
         __lowerCamelCase  :      Optional[Any]														=						eos_token_id

         super().__init__(
             tie_word_embeddings=SCREAMING_SNAKE_CASE_     ,	bos_token_id=SCREAMING_SNAKE_CASE_     ,	eos_token_id=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )






