
'''simple docstring'''






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase_							: Optional[Any] =							logging.get_logger(__name__)







def 		A_							(						_lowerCAmelCase :     Optional[Any]					,						_lowerCAmelCase :     Tuple=False				):



      """simple docstring"""




      _lowerCamelCase					: Union[str, Any]		    = []
      for i in range(config.num_hidden_layers				):
            # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
            rename_keys.append((F'blocks.{i}.norm1.weight', F'vit.encoder.layer.{i}.layernorm_before.weight')				)
            rename_keys.append((F'blocks.{i}.norm1.bias', F'vit.encoder.layer.{i}.layernorm_before.bias')				)
            rename_keys.append((F'blocks.{i}.attn.proj.weight', F'vit.encoder.layer.{i}.attention.output.dense.weight')				)
            rename_keys.append((F'blocks.{i}.attn.proj.bias', F'vit.encoder.layer.{i}.attention.output.dense.bias')				)
            rename_keys.append((F'blocks.{i}.norm2.weight', F'vit.encoder.layer.{i}.layernorm_after.weight')				)
            rename_keys.append((F'blocks.{i}.norm2.bias', F'vit.encoder.layer.{i}.layernorm_after.bias')				)
            rename_keys.append((F'blocks.{i}.mlp.fc1.weight', F'vit.encoder.layer.{i}.intermediate.dense.weight')				)
            rename_keys.append((F'blocks.{i}.mlp.fc1.bias', F'vit.encoder.layer.{i}.intermediate.dense.bias')				)
            rename_keys.append((F'blocks.{i}.mlp.fc2.weight', F'vit.encoder.layer.{i}.output.dense.weight')				)
            rename_keys.append((F'blocks.{i}.mlp.fc2.bias', F'vit.encoder.layer.{i}.output.dense.bias')				)

      # projection layer + position embeddings
      rename_keys.extend(
          [
              ("cls_token", "vit.embeddings.cls_token"),
              ("patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight"),
              ("patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias"),
              ("pos_embed", "vit.embeddings.position_embeddings"),
          ]				)

      if base_model:
            # layernorm + pooler
            rename_keys.extend(
                [
                    ("norm.weight", "layernorm.weight"),
                    ("norm.bias", "layernorm.bias"),
                ]				)

            # if just the base model, we should remove "vit" from all keys that start with "vit"
            _lowerCamelCase					: int		    = [(pair[0], pair[1][4:]) if pair[1].startswith("vit"				) else pair for pair in rename_keys]
      else:
            # layernorm + classification head
            rename_keys.extend(
                [
                    ("norm.weight", "vit.layernorm.weight"),
                    ("norm.bias", "vit.layernorm.bias"),
                    ("head.weight", "classifier.weight"),
                    ("head.bias", "classifier.bias"),
                ]				)

      return rename_keys







def 		A_							(						_lowerCAmelCase :     int					,						_lowerCAmelCase :     Any					,						_lowerCAmelCase :     int=False				):



      """simple docstring"""




      for i in range(config.num_hidden_layers				):
            if base_model:
                  _lowerCamelCase					: Any		    = ""
            else:
                  _lowerCamelCase					: List[Any]		    = "vit."
            # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
            _lowerCamelCase					: Tuple		    = state_dict.pop(F'blocks.{i}.attn.qkv.weight'				)
            _lowerCamelCase					: int		    = state_dict.pop(F'blocks.{i}.attn.qkv.bias'				)
            # next, add query, keys and values (in that order) to the state dict
            _lowerCamelCase					: Optional[int]		    = in_proj_weight[
                : config.hidden_size, :
            ]
            _lowerCamelCase					: Any		    = in_proj_bias[: config.hidden_size]
            _lowerCamelCase					: Any		    = in_proj_weight[
                config.hidden_size : config.hidden_size * 2, :
            ]
            _lowerCamelCase					: Union[str, Any]		    = in_proj_bias[
                config.hidden_size : config.hidden_size * 2
            ]
            _lowerCamelCase					: str		    = in_proj_weight[
                -config.hidden_size :, :
            ]
            _lowerCamelCase					: Optional[int]		    = in_proj_bias[-config.hidden_size :]







def 		A_							(						_lowerCAmelCase :     str				):



      """simple docstring"""




      _lowerCamelCase					: Union[str, Any]		    = ["head.weight", "head.bias"]
      for k in ignore_keys:
            state_dict.pop(_lowerCAmelCase					,						_lowerCAmelCase				)







def 		A_							(						_lowerCAmelCase :     Optional[int]					,						_lowerCAmelCase :     Dict					,						_lowerCAmelCase :     List[str]				):



      """simple docstring"""




      _lowerCamelCase					: str		    = dct.pop(_lowerCAmelCase				)
      _lowerCamelCase					: str		    = val







def 		A_							(						):



      """simple docstring"""




      _lowerCamelCase					: Optional[Any]		    = "http://images.cocodataset.org/val2017/000000039769.jpg"
      _lowerCamelCase					: int		    = Image.open(requests.get(_lowerCAmelCase					,						stream=_lowerCAmelCase				).raw				)
      return im







@torch.no_grad()
def 		A_							(						_lowerCAmelCase :     Optional[Any]					,						_lowerCAmelCase :     Optional[int]					,						_lowerCAmelCase :     Any=True				):



      """simple docstring"""




      _lowerCamelCase					: Dict		    = ViTConfig()
      # patch_size
      if model_name[-1] == "8":
            _lowerCamelCase					: Dict		    = 8
      # set labels if required
      if not base_model:
            _lowerCamelCase					: Tuple		    = 1000
            _lowerCamelCase					: List[str]		    = "huggingface/label-files"
            _lowerCamelCase					: Optional[int]		    = "imagenet-1k-id2label.json"
            _lowerCamelCase					: Tuple		    = json.load(open(hf_hub_download(_lowerCAmelCase					,						_lowerCAmelCase					,						repo_type="dataset"				)					,						"r"				)				)
            _lowerCamelCase					: Tuple		    = {int(_lowerCAmelCase				): v for k, v in idalabel.items()}
            _lowerCamelCase					: Dict		    = idalabel
            _lowerCamelCase					: List[str]		    = {v: k for k, v in idalabel.items()}
      # size of the architecture
      if model_name in ["dino_vits8", "dino_vits16"]:
            _lowerCamelCase					: Any		    = 384
            _lowerCamelCase					: Union[str, Any]		    = 1536
            _lowerCamelCase					: Union[str, Any]		    = 12
            _lowerCamelCase					: Optional[int]		    = 6

      # load original model from torch hub
      _lowerCamelCase					: Optional[int]		    = torch.hub.load("facebookresearch/dino:main"					,						_lowerCAmelCase				)
      original_model.eval()

      # load state_dict of original model, remove and rename some keys
      _lowerCamelCase					: Dict		    = original_model.state_dict()
      if base_model:
            remove_classification_head_(_lowerCAmelCase				)
      _lowerCamelCase					: Dict		    = create_rename_keys(_lowerCAmelCase					,						base_model=_lowerCAmelCase				)
      for src, dest in rename_keys:
            rename_key(_lowerCAmelCase					,						_lowerCAmelCase					,						_lowerCAmelCase				)
      read_in_q_k_v(_lowerCAmelCase					,						_lowerCAmelCase					,						_lowerCAmelCase				)

      # load HuggingFace model
      if base_model:
            _lowerCamelCase					: Optional[int]		    = ViTModel(_lowerCAmelCase					,						add_pooling_layer=_lowerCAmelCase				).eval()
      else:
            _lowerCamelCase					: Optional[int]		    = ViTForImageClassification(_lowerCAmelCase				).eval()
      model.load_state_dict(_lowerCAmelCase				)

      # Check outputs on an image, prepared by ViTImageProcessor
      _lowerCamelCase					: Optional[int]		    = ViTImageProcessor()
      _lowerCamelCase					: List[Any]		    = image_processor(images=prepare_img()					,						return_tensors="pt"				)
      _lowerCamelCase					: Optional[Any]		    = encoding["pixel_values"]
      _lowerCamelCase					: Any		    = model(_lowerCAmelCase				)

      if base_model:
            _lowerCamelCase					: Optional[int]		    = original_model(_lowerCAmelCase				)
            assert torch.allclose(_lowerCAmelCase					,						outputs.last_hidden_state[:, 0, :]					,						atol=1E-1				)
      else:
            _lowerCamelCase					: List[str]		    = original_model(_lowerCAmelCase				)
            assert logits.shape == outputs.logits.shape
            assert torch.allclose(_lowerCAmelCase					,						outputs.logits					,						atol=1E-3				)

      Path(_lowerCAmelCase				).mkdir(exist_ok=_lowerCAmelCase				)
      print(F'Saving model {model_name} to {pytorch_dump_folder_path}'				)
      model.save_pretrained(_lowerCAmelCase				)
      print(F'Saving image processor to {pytorch_dump_folder_path}'				)
      image_processor.save_pretrained(_lowerCAmelCase				)


if __name__ == "__main__":
 UpperCAmelCase_							: str =							argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--model_name',
     default='dino_vitb16',
     type=str,
     help='Name of the model trained with DINO you\'d like to convert.',
 )
 parser.add_argument(
     '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
 )
 parser.add_argument(
     '--base_model',
     action='store_true',
     help='Whether to only convert the base model (no projection head weights).',
 )

 parser.set_defaults(base_model=True)
 UpperCAmelCase_							: str =							parser.parse_args()
 convert_vit_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.base_model)






'''simple docstring'''






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
A__							:  List[str]    =logging.getLogger(__name__)
def   UpperCamelCase__ (						):





      """simple docstring"""


      _lowerCAmelCase					=				argparse.ArgumentParser(
          description="""Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."""    )
      parser.add_argument("""--file_path"""				,      type=lowerCAmelCase				,      default="""data/dump.txt"""				,      help="""The path to the data."""    )
      parser.add_argument("""--tokenizer_type"""				,      type=lowerCAmelCase				,      default="""bert"""				,      choices=["""bert""", """roberta""", """gpt2"""]    )
      parser.add_argument("""--tokenizer_name"""				,      type=lowerCAmelCase				,      default="""bert-base-uncased"""				,      help="""The tokenizer to use."""    )
      parser.add_argument("""--dump_file"""				,      type=lowerCAmelCase				,      default="""data/dump"""				,      help="""The dump file prefix."""    )
      _lowerCAmelCase					=				parser.parse_args()

      logger.info(f"Loading Tokenizer ({args.tokenizer_name})"    )
      if args.tokenizer_type == "bert":
            _lowerCAmelCase					=				BertTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""cls_token"""]  # `[CLS]`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""sep_token"""]  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            _lowerCAmelCase					=				RobertaTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""cls_token"""]  # `<s>`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""sep_token"""]  # `</s>`
      elif args.tokenizer_type == "gpt2":
            _lowerCAmelCase					=				GPTaTokenizer.from_pretrained(args.tokenizer_name    )
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""bos_token"""]  # `<|endoftext|>`
            _lowerCAmelCase					=				tokenizer.special_tokens_map["""eos_token"""]  # `<|endoftext|>`

      logger.info(f"Loading text from {args.file_path}"    )
      with open(args.file_path				,      """r"""				,      encoding="""utf8"""    ) as fp:
            _lowerCAmelCase					=				fp.readlines()

      logger.info("""Start encoding"""    )
      logger.info(f"{len(lowerCAmelCase    )} examples to process."    )

      _lowerCAmelCase					=				[]
      _lowerCAmelCase					=				0
      _lowerCAmelCase					=				1_00_00
      _lowerCAmelCase					=				time.time()
      for text in data:
            _lowerCAmelCase					=				f"{bos} {text.strip()} {sep}"
            _lowerCAmelCase					=				tokenizer.encode(lowerCAmelCase				,      add_special_tokens=lowerCAmelCase    )
            rslt.append(lowerCAmelCase    )

            iter += 1
            if iter % interval == 0:
                  _lowerCAmelCase					=				time.time()
                  logger.info(f"{iter} examples processed. - {(end-start):.2f}s/{interval}expl"    )
                  _lowerCAmelCase					=				time.time()
      logger.info("""Finished binarization"""    )
      logger.info(f"{len(lowerCAmelCase    )} examples processed."    )

      _lowerCAmelCase					=				f"{args.dump_file}.{args.tokenizer_name}.pickle"
      _lowerCAmelCase					=				tokenizer.vocab_size
      if vocab_size < (1 << 16):
            _lowerCAmelCase					=				[np.uintaa(lowerCAmelCase    ) for d in rslt]
      else:
            _lowerCAmelCase					=				[np.intaa(lowerCAmelCase    ) for d in rslt]
      random.shuffle(rslt_    )
      logger.info(f"Dump to {dp_file}"    )
      with open(lowerCAmelCase				,      """wb"""    ) as handle:
            pickle.dump(rslt_				,      lowerCAmelCase				,      protocol=pickle.HIGHEST_PROTOCOL    )


if __name__ == "__main__":
      main()






