






import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import datasets
import numpy as np
import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    EvalPrediction,
    HfArgumentParser,
    PreTrainedTokenizer,
    TFAutoModelForSequenceClassification,
    TFTrainer,
    TFTrainingArguments,
)
from transformers.utils import logging as hf_logging


hf_logging.set_verbosity_info()
hf_logging.enable_default_handler()
hf_logging.enable_explicit_format()
def 		_a       ( a					:str      ,				a					:str      ,				a					:str      ,				a					:PreTrainedTokenizer      ,				a					:int      ,				a					:Optional[int] = None      ,				)				->     int:
 a						    = {}

 if train_file is not None:
  a						    = [train_file]
 if eval_file is not None:
  a						    = [eval_file]
 if test_file is not None:
  a						    = [test_file]

 a						    = datasets.load_dataset('''csv'''      ,				data_files=a					)
 a						    = list(ds[list(files.keys()					)[0]].features.keys()					)
 a						    = features_name.pop(a					)
 a						    = list(set(ds[list(files.keys()					)[0]][label_name]					)					)
 a						    = {label: i for i, label in enumerate(a					)}
 a						    = tokenizer.model_input_names
 a						    = {}

 if len(a					) == 1:
  for k in files.keys():
   a						    = ds[k].map(
       lambda a					: tokenizer.batch_encode_plus(
           example[features_name[0]]      ,				truncation=a      ,				max_length=a      ,				padding='''max_length'''					)      ,				batched=a      ,				)
 elif len(a					) == 2:
  for k in files.keys():
   a						    = ds[k].map(
       lambda a					: tokenizer.batch_encode_plus(
           (example[features_name[0]], example[features_name[1]])      ,				truncation=a      ,				max_length=a      ,				padding='''max_length'''      ,				)      ,				batched=a      ,				)

 def gen_train():
  for ex in transformed_ds[datasets.Split.TRAIN]:
   a						    = {k: v for k, v in ex.items() if k in input_names}
   a						    = labelaid[ex[label_name]]
   yield (d, label)

 def gen_val():
  for ex in transformed_ds[datasets.Split.VALIDATION]:
   a						    = {k: v for k, v in ex.items() if k in input_names}
   a						    = labelaid[ex[label_name]]
   yield (d, label)

 def gen_test():
  for ex in transformed_ds[datasets.Split.TEST]:
   a						    = {k: v for k, v in ex.items() if k in input_names}
   a						    = labelaid[ex[label_name]]
   yield (d, label)

 a						    = (
     tf.data.Dataset.from_generator(
         a      ,				({k: tf.intaa for k in input_names}, tf.intaa)      ,				({k: tf.TensorShape([None]					) for k in input_names}, tf.TensorShape([]					))      ,				)
     if datasets.Split.TRAIN in transformed_ds
     else None
 )

 if train_ds is not None:
  a						    = train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN]					)					)					)

 a						    = (
     tf.data.Dataset.from_generator(
         a      ,				({k: tf.intaa for k in input_names}, tf.intaa)      ,				({k: tf.TensorShape([None]					) for k in input_names}, tf.TensorShape([]					))      ,				)
     if datasets.Split.VALIDATION in transformed_ds
     else None
 )

 if val_ds is not None:
  a						    = val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION]					)					)					)

 a						    = (
     tf.data.Dataset.from_generator(
         a      ,				({k: tf.intaa for k in input_names}, tf.intaa)      ,				({k: tf.TensorShape([None]					) for k in input_names}, tf.TensorShape([]					))      ,				)
     if datasets.Split.TEST in transformed_ds
     else None
 )

 if test_ds is not None:
  a						    = test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST]					)					)					)

 return train_ds, val_ds, test_ds, labelaid


UpperCAmelCase__							  =						logging.getLogger(__name__)






@dataclass
class 						lowercase_   :






      '''simple docstring'''




      __snake_case	    =    field(metadata={'''help''': '''Which column contains the label'''}  )
      __snake_case	    =    field(default=lowercase					,    metadata={'''help''': '''The path of the training file'''}  )
      __snake_case	    =    field(default=lowercase					,    metadata={'''help''': '''The path of the development file'''}  )
      __snake_case	    =    field(default=lowercase					,    metadata={'''help''': '''The path of the test file'''}  )
      __snake_case	    =    field(
          default=1_28					,    metadata={
              '''help''': (
                  '''The maximum total input sequence length after tokenization. Sequences longer '''
                  '''than this will be truncated, sequences shorter will be padded.'''
              )
          }					,    )
      __snake_case	    =    field(
          default=lowercase					,    metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}  )






@dataclass
class 						lowercase_   :






      '''simple docstring'''




      __snake_case	    =    field(
          metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}  )
      __snake_case	    =    field(
          default=lowercase					,    metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}  )
      __snake_case	    =    field(
          default=lowercase					,    metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}  )
      __snake_case	    =    field(default=lowercase					,    metadata={'''help''': '''Set this flag to use fast tokenization.'''}  )
      # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
      # or just modify its tokenizer_config.json.
      __snake_case	    =    field(
          default=lowercase					,    metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}					,    )
def 		_a       ( )				->     Union[str, Any]:
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.
 a						    = HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)					)
 a    ,	a    ,	a						    = parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir					)
     and os.listdir(training_args.output_dir					)
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
      ''' --overwrite_output_dir to overcome.'''					)

 # Setup logging
 logging.basicConfig(
     format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''      ,				datefmt='''%m/%d/%Y %H:%M:%S'''      ,				level=logging.INFO      ,				)
 logger.info(
     F"""n_replicas: {training_args.n_replicas}, distributed training: {bool(training_args.n_replicas > 1					)}, """
     F"""16-bits training: {training_args.fpaa}"""					)
 logger.info(F"""Training/evaluation parameters {training_args}"""					)

 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 a						    = AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path      ,				cache_dir=model_args.cache_dir      ,				)

 a    ,	a    ,	a    ,	a						    = get_tfds(
     train_file=data_args.train_file      ,				eval_file=data_args.dev_file      ,				test_file=data_args.test_file      ,				tokenizer=a      ,				label_column_id=data_args.label_column_id      ,				max_seq_length=data_args.max_seq_length      ,				)

 a						    = AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path      ,				num_labels=len(a					)      ,				labelaid=a      ,				idalabel={id: label for label, id in labelaid.items()}      ,				finetuning_task='''text-classification'''      ,				cache_dir=model_args.cache_dir      ,				)

 with training_args.strategy.scope():
  a						    = TFAutoModelForSequenceClassification.from_pretrained(
      model_args.model_name_or_path      ,				from_pt=bool('''.bin''' in model_args.model_name_or_path					)      ,				config=a      ,				cache_dir=model_args.cache_dir      ,				)

 def compute_metrics(a					:EvalPrediction					) -> Dict:
  a						    = np.argmax(p.predictions      ,				axis=1					)

  return {"acc": (preds == p.label_ids).mean()}

 # Initialize our Trainer
 a						    = TFTrainer(
     model=a      ,				args=a      ,				train_dataset=a      ,				eval_dataset=a      ,				compute_metrics=a      ,				)

 # Training
 if training_args.do_train:
  trainer.train()
  trainer.save_model()
  tokenizer.save_pretrained(training_args.output_dir					)

 # Evaluation
 a						    = {}
 if training_args.do_eval:
  logger.info('''*** Evaluate ***'''					)
  a						    = trainer.evaluate()
  a						    = os.path.join(training_args.output_dir      ,				'''eval_results.txt'''					)

  with open(a      ,				'''w'''					) as writer:
   logger.info('''***** Eval results *****'''					)

   for key, value in result.items():
    logger.info(F"""  {key} = {value}"""					)
    writer.write(F"""{key} = {value}\n"""					)

   results.update(a					)

 return results


if __name__ == "__main__":
     main()










import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase__							  =						logging.get_logger(__name__)
def 		_a       ( a					:List[str]      ,				a					:Tuple=False					)				->     List[Any]:
 a						    = []
 for i in range(config.num_hidden_layers					):
  # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
  rename_keys.append((F"""blocks.{i}.norm1.weight""", F"""vit.encoder.layer.{i}.layernorm_before.weight""")					)
  rename_keys.append((F"""blocks.{i}.norm1.bias""", F"""vit.encoder.layer.{i}.layernorm_before.bias""")					)
  rename_keys.append((F"""blocks.{i}.attn.proj.weight""", F"""vit.encoder.layer.{i}.attention.output.dense.weight""")					)
  rename_keys.append((F"""blocks.{i}.attn.proj.bias""", F"""vit.encoder.layer.{i}.attention.output.dense.bias""")					)
  rename_keys.append((F"""blocks.{i}.norm2.weight""", F"""vit.encoder.layer.{i}.layernorm_after.weight""")					)
  rename_keys.append((F"""blocks.{i}.norm2.bias""", F"""vit.encoder.layer.{i}.layernorm_after.bias""")					)
  rename_keys.append((F"""blocks.{i}.mlp.fc1.weight""", F"""vit.encoder.layer.{i}.intermediate.dense.weight""")					)
  rename_keys.append((F"""blocks.{i}.mlp.fc1.bias""", F"""vit.encoder.layer.{i}.intermediate.dense.bias""")					)
  rename_keys.append((F"""blocks.{i}.mlp.fc2.weight""", F"""vit.encoder.layer.{i}.output.dense.weight""")					)
  rename_keys.append((F"""blocks.{i}.mlp.fc2.bias""", F"""vit.encoder.layer.{i}.output.dense.bias""")					)

 # projection layer + position embeddings
 rename_keys.extend(
     [
         ('''cls_token''', '''vit.embeddings.cls_token'''),
         ('''patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight'''),
         ('''patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias'''),
         ('''pos_embed''', '''vit.embeddings.position_embeddings'''),
     ]					)

 if base_model:
  # layernorm + pooler
  rename_keys.extend(
      [
          ('''norm.weight''', '''layernorm.weight'''),
          ('''norm.bias''', '''layernorm.bias'''),
      ]					)

  # if just the base model, we should remove "vit" from all keys that start with "vit"
  a						    = [(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''					) else pair for pair in rename_keys]
 else:
  # layernorm + classification head
  rename_keys.extend(
      [
          ('''norm.weight''', '''vit.layernorm.weight'''),
          ('''norm.bias''', '''vit.layernorm.bias'''),
          ('''head.weight''', '''classifier.weight'''),
          ('''head.bias''', '''classifier.bias'''),
      ]					)

 return rename_keys
def 		_a       ( a					:List[str]      ,				a					:int      ,				a					:Tuple=False					)				->     Any:
 for i in range(config.num_hidden_layers					):
  if base_model:
   a						    = ''''''
  else:
   a						    = '''vit.'''
  # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
  a						    = state_dict.pop(F"""blocks.{i}.attn.qkv.weight"""					)
  a						    = state_dict.pop(F"""blocks.{i}.attn.qkv.bias"""					)
  # next, add query, keys and values (in that order) to the state dict
  a						    = in_proj_weight[
      : config.hidden_size, :
  ]
  a						    = in_proj_bias[: config.hidden_size]
  a						    = in_proj_weight[
      config.hidden_size : config.hidden_size * 2, :
  ]
  a						    = in_proj_bias[
      config.hidden_size : config.hidden_size * 2
  ]
  a						    = in_proj_weight[
      -config.hidden_size :, :
  ]
  a						    = in_proj_bias[-config.hidden_size :]
def 		_a       ( a					:List[Any]					)				->     Dict:
 a						    = ['''head.weight''', '''head.bias''']
 for k in ignore_keys:
  state_dict.pop(a      ,				a					)
def 		_a       ( a					:Union[str, Any]      ,				a					:str      ,				a					:List[Any]					)				->     str:
 a						    = dct.pop(a					)
 a						    = val
def 		_a       ( )				->     Optional[Any]:
 a						    = '''http://images.cocodataset.org/val2017/000000039769.jpg'''
 a						    = Image.open(requests.get(a      ,				stream=a					).raw					)
 return im
@torch.no_grad()
def 		_a       ( a					:Tuple      ,				a					:str      ,				a					:Optional[int]=True					)				->     Dict:
 a						    = ViTConfig()
 # patch_size
 if model_name[-1] == "8":
  a						    = 8
 # set labels if required
 if not base_model:
  a						    = 1_000
  a						    = '''huggingface/label-files'''
  a						    = '''imagenet-1k-id2label.json'''
  a						    = json.load(open(hf_hub_download(a      ,				a      ,				repo_type='''dataset'''					)      ,				'''r'''					)					)
  a						    = {int(a					): v for k, v in idalabel.items()}
  a						    = idalabel
  a						    = {v: k for k, v in idalabel.items()}
 # size of the architecture
 if model_name in ["dino_vits8", "dino_vits16"]:
  a						    = 384
  a						    = 1_536
  a						    = 12
  a						    = 6

 # load original model from torch hub
 a						    = torch.hub.load('''facebookresearch/dino:main'''      ,				a					)
 original_model.eval()

 # load state_dict of original model, remove and rename some keys
 a						    = original_model.state_dict()
 if base_model:
  remove_classification_head_(a					)
 a						    = create_rename_keys(a      ,				base_model=a					)
 for src, dest in rename_keys:
  rename_key(a      ,				a      ,				a					)
 read_in_q_k_v(a      ,				a      ,				a					)

 # load HuggingFace model
 if base_model:
  a						    = ViTModel(a      ,				add_pooling_layer=a					).eval()
 else:
  a						    = ViTForImageClassification(a					).eval()
 model.load_state_dict(a					)

 # Check outputs on an image, prepared by ViTImageProcessor
 a						    = ViTImageProcessor()
 a						    = image_processor(images=prepare_img()      ,				return_tensors='''pt'''					)
 a						    = encoding['''pixel_values''']
 a						    = model(a					)

 if base_model:
  a						    = original_model(a					)
  assert torch.allclose(a      ,				outputs.last_hidden_state[:, 0, :]      ,				atol=1e-1					)
 else:
  a						    = original_model(a					)
  assert logits.shape == outputs.logits.shape
  assert torch.allclose(a      ,				outputs.logits      ,				atol=1e-3					)

 Path(a					).mkdir(exist_ok=a					)
 print(F"""Saving model {model_name} to {pytorch_dump_folder_path}"""					)
 model.save_pretrained(a					)
 print(F"""Saving image processor to {pytorch_dump_folder_path}"""					)
 image_processor.save_pretrained(a					)


if __name__ == "__main__":
     UpperCAmelCase__							  =						argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         "--model_name",
         default="dino_vitb16",
         type=str,
         help="Name of the model trained with DINO you'd like to convert.",
     )
     parser.add_argument(
         "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
     )
     parser.add_argument(
         "--base_model",
         action="store_true",
         help="Whether to only convert the base model (no projection head weights).",
     )

     parser.set_defaults(base_model=True)
     UpperCAmelCase__							  =						parser.parse_args()
     convert_vit_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.base_model)


