


import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTImageProcessor, ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
_lowerCAmelCase:						str   				=       logging.get_logger(__name__)






def   _lowercase(					__a				:						List[str]    ,     __a				:						List[str]=False     ):
							a__      =[]
							for i in range(config.num_hidden_layers     ):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((f"""blocks.{i}.norm1.weight""", f"""vit.encoder.layer.{i}.layernorm_before.weight""")     )
														rename_keys.append((f"""blocks.{i}.norm1.bias""", f"""vit.encoder.layer.{i}.layernorm_before.bias""")     )
														rename_keys.append((f"""blocks.{i}.attn.proj.weight""", f"""vit.encoder.layer.{i}.attention.output.dense.weight""")     )
														rename_keys.append((f"""blocks.{i}.attn.proj.bias""", f"""vit.encoder.layer.{i}.attention.output.dense.bias""")     )
														rename_keys.append((f"""blocks.{i}.norm2.weight""", f"""vit.encoder.layer.{i}.layernorm_after.weight""")     )
														rename_keys.append((f"""blocks.{i}.norm2.bias""", f"""vit.encoder.layer.{i}.layernorm_after.bias""")     )
														rename_keys.append((f"""blocks.{i}.mlp.fc1.weight""", f"""vit.encoder.layer.{i}.intermediate.dense.weight""")     )
														rename_keys.append((f"""blocks.{i}.mlp.fc1.bias""", f"""vit.encoder.layer.{i}.intermediate.dense.bias""")     )
														rename_keys.append((f"""blocks.{i}.mlp.fc2.weight""", f"""vit.encoder.layer.{i}.output.dense.weight""")     )
														rename_keys.append((f"""blocks.{i}.mlp.fc2.bias""", f"""vit.encoder.layer.{i}.output.dense.bias""")     )

							# projection layer + position embeddings
							rename_keys.extend(
							    [
							        ('cls_token', 'vit.embeddings.cls_token'),
							        ('patch_embed.proj.weight', 'vit.embeddings.patch_embeddings.projection.weight'),
							        ('patch_embed.proj.bias', 'vit.embeddings.patch_embeddings.projection.bias'),
							        ('pos_embed', 'vit.embeddings.position_embeddings'),
							    ]     )

							if base_model:
														# layernorm + pooler
														rename_keys.extend(
														    [
														        ('norm.weight', 'layernorm.weight'),
														        ('norm.bias', 'layernorm.bias'),
														        ('pre_logits.fc.weight', 'pooler.dense.weight'),
														        ('pre_logits.fc.bias', 'pooler.dense.bias'),
														    ]     )

														# if just the base model, we should remove "vit" from all keys that start with "vit"
														a__      =[(pair[0], pair[1][4:]) if pair[1].startswith('vit'     ) else pair for pair in rename_keys]
							else:
														# layernorm + classification head
														rename_keys.extend(
														    [
														        ('norm.weight', 'vit.layernorm.weight'),
														        ('norm.bias', 'vit.layernorm.bias'),
														        ('head.weight', 'classifier.weight'),
														        ('head.bias', 'classifier.bias'),
														    ]     )

							return rename_keys






def   _lowercase(					__a				:						List[Any]    ,     __a				:						List[Any]    ,     __a				:						Union[str, Any]=False     ):
							for i in range(config.num_hidden_layers     ):
														if base_model:
																					a__      =''
														else:
																					a__      ='vit.'
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														a__      =state_dict.pop(f"""blocks.{i}.attn.qkv.weight"""     )
														a__      =state_dict.pop(f"""blocks.{i}.attn.qkv.bias"""     )
														# next, add query, keys and values (in that order) to the state dict
														a__      =in_proj_weight[
														    : config.hidden_size, :
														]
														a__      =in_proj_bias[: config.hidden_size]
														a__      =in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														a__      =in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														a__      =in_proj_weight[
														    -config.hidden_size :, :
														]
														a__      =in_proj_bias[-config.hidden_size :]






def   _lowercase(					__a				:						Optional[Any]     ):
							a__      =['head.weight', 'head.bias']
							for k in ignore_keys:
														state_dict.pop(__a    ,     __a     )






def   _lowercase(					__a				:						List[Any]    ,     __a				:						List[Any]    ,     __a				:						List[str]     ):
							a__      =dct.pop(__a     )
							a__      =val






def   _lowercase(					):
							a__      ='http://images.cocodataset.org/val2017/000000039769.jpg'
							a__      =Image.open(requests.get(__a    ,     stream=__a     ).raw     )
							return im






@torch.no_grad()
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						List[Any]     ):
							a__      =ViTConfig()
							a__      =False
							# dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
							if vit_name[-5:] == "in21k":
														a__      =True
														a__      =int(vit_name[-12:-10]     )
														a__      =int(vit_name[-9:-6]     )
							else:
														a__      =1000
														a__      ='huggingface/label-files'
														a__      ='imagenet-1k-id2label.json'
														a__      =json.load(open(hf_hub_download(__a    ,     __a    ,     repo_type='dataset'     )    ,     'r'     )     )
														a__      ={int(__a     ): v for k, v in idalabel.items()}
														a__      =idalabel
														a__      ={v: k for k, v in idalabel.items()}
														a__      =int(vit_name[-6:-4]     )
														a__      =int(vit_name[-3:]     )
							# size of the architecture
							if "deit" in vit_name:
														if vit_name[9:].startswith('tiny'     ):
																					a__      =192
																					a__      =768
																					a__      =12
																					a__      =3
														elif vit_name[9:].startswith('small'     ):
																					a__      =384
																					a__      =1536
																					a__      =12
																					a__      =6
														else:
																					pass
							else:
														if vit_name[4:].startswith('small'     ):
																					a__      =768
																					a__      =2304
																					a__      =8
																					a__      =8
														elif vit_name[4:].startswith('base'     ):
																					pass
														elif vit_name[4:].startswith('large'     ):
																					a__      =1024
																					a__      =4096
																					a__      =24
																					a__      =16
														elif vit_name[4:].startswith('huge'     ):
																					a__      =1280
																					a__      =5120
																					a__      =32
																					a__      =16

    # load original model from timm
							a__      =timm.create_model(__a    ,     pretrained=__a     )
							timm_model.eval()

							# load state_dict of original model, remove and rename some keys
							a__      =timm_model.state_dict()
							if base_model:
														remove_classification_head_(__a     )
							a__      =create_rename_keys(__a    ,     __a     )
							for src, dest in rename_keys:
														rename_key(__a    ,     __a    ,     __a     )
							read_in_q_k_v(__a    ,     __a    ,     __a     )

							# load HuggingFace model
							if vit_name[-5:] == "in21k":
														a__      =ViTModel(__a     ).eval()
							else:
														a__      =ViTForImageClassification(__a     ).eval()
							model.load_state_dict(__a     )

							# Check outputs on an image, prepared by ViTImageProcessor/DeiTImageProcessor
							if "deit" in vit_name:
														a__      =DeiTImageProcessor(size=config.image_size     )
							else:
														a__      =ViTImageProcessor(size=config.image_size     )
							a__      =image_processor(images=prepare_img()    ,     return_tensors='pt'     )
							a__      =encoding['pixel_values']
							a__      =model(__a     )

							if base_model:
														a__      =timm_model.forward_features(__a     )
														assert timm_pooled_output.shape == outputs.pooler_output.shape
														assert torch.allclose(__a    ,     outputs.pooler_output    ,     atol=1e-3     )
							else:
														a__      =timm_model(__a     )
														assert timm_logits.shape == outputs.logits.shape
														assert torch.allclose(__a    ,     outputs.logits    ,     atol=1e-3     )

							Path(__a     ).mkdir(exist_ok=__a     )
							print(f"""Saving model {vit_name} to {pytorch_dump_folder_path}"""     )
							model.save_pretrained(__a     )
							print(f"""Saving image processor to {pytorch_dump_folder_path}"""     )
							image_processor.save_pretrained(__a     )


if __name__ == "__main__":
	_lowerCAmelCase:						str   				=       argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--vit_name',
	    default='vit_base_patch16_224',
	    type=str,
	    help='Name of the ViT timm model you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
	)

	_lowerCAmelCase:						Optional[int]   				=       parser.parse_args()
	convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path)



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A							=logging.getLogger(__name__)





def      lowerCamelCase_					(							):
				lowerCamelCase_         =     argparse.ArgumentParser(
				    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."       )
				parser.add_argument(
				    "--dataset_name"      ,					type=lowerCamelCase__      ,					default="wikitext"      ,					help="Name of the training. Explore datasets at: hf.co/datasets."      ,					)
				parser.add_argument(
				    "--dataset_config"      ,					type=lowerCamelCase__      ,					default="wikitext-103-raw-v1"      ,					help="Configuration name of the dataset."       )
				parser.add_argument(
				    "--tokenizer_name_or_path"      ,					type=lowerCamelCase__      ,					default="sayakpaul/unigram-tokenizer-wikitext"      ,					help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,					)
				parser.add_argument(
				    "--shard_size"      ,					type=lowerCamelCase__      ,					default=1_0_0_0      ,					help="Number of entries to go in a single shard."      ,					)
				parser.add_argument("--split"      ,					type=lowerCamelCase__      ,					default="train"      ,					choices=["train", "test", "validation"]       )
				parser.add_argument(
				    "--limit"      ,					default=lowerCamelCase__      ,					type=lowerCamelCase__      ,					help="Limit the number of shards (used for debugging)."      ,					)
				parser.add_argument(
				    "--max_length"      ,					type=lowerCamelCase__      ,					default=5_1_2      ,					help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
				    " sequence length that is a multiple of 8."      ,					)
				parser.add_argument(
				    "--output_dir"      ,					default="tf-tpu"      ,					type=lowerCamelCase__      ,					help="Output directory where the TFRecord shards will be saved. If the"
				    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
				    " shards will be directly saved to a Google Cloud Storage bucket."      ,					)

				lowerCamelCase_         =     parser.parse_args()
				return args





def      lowerCamelCase_					(							lowerCamelCase__       ):
				def fn(lowerCamelCase__       ):
								return tokenizer(examples["text"]       )

				return fn





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     []
				for i in range(len(tokenized_data["input_ids"]       )       ):
								lowerCamelCase_         =     {
								    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]       )       ),
								    "attention_mask": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]       )       ),
								}
								lowerCamelCase_         =     tf.train.Features(feature=lowerCamelCase__       )
								lowerCamelCase_         =     tf.train.Example(features=lowerCamelCase__       )
								lowerCamelCase_         =     example.SerializeToString()
								records.append(lowerCamelCase__       )
				return records





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     datasets.load_dataset(args.dataset_name      ,					args.dataset_config      ,					split=args.split       )

				if args.limit is not None:
								lowerCamelCase_         =     min(len(lowerCamelCase__       )      ,					args.limit       )
								lowerCamelCase_         =     dataset.select(range(lowerCamelCase__       )       )
								print(F'Limiting the dataset to {args.limit} entries.'       )

				lowerCamelCase_         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir       ):
												os.makedirs(args.output_dir       )
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )
								if not os.path.exists(lowerCamelCase__       ):
												os.makedirs(lowerCamelCase__       )
				else:
								lowerCamelCase_         =     os.path.join(args.output_dir      ,					args.split       )

				# Tokenize the whole dataset at once.
				lowerCamelCase_         =     tokenize_function(lowerCamelCase__       )
				lowerCamelCase_         =     dataset.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					num_proc=4      ,					remove_columns=["text"]       )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowerCamelCase__       ):
								# Concatenate all texts.
								lowerCamelCase_         =     {k: sum(examples[k]      ,					[]       ) for k in examples.keys()}
								lowerCamelCase_         =     len(concatenated_examples[list(examples.keys()       )[0]]       )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowerCamelCase_         =     (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowerCamelCase_         =     {
								    k: [t[i : i + args.max_length] for i in range(0      ,					lowerCamelCase__      ,					args.max_length       )]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowerCamelCase_         =     dataset_tokenized.map(lowerCamelCase__      ,					batched=lowerCamelCase__      ,					batch_size=1_0_0_0      ,					num_proc=4       )

				lowerCamelCase_         =     0
				lowerCamelCase_         =     0
				for shard in range(0      ,					len(lowerCamelCase__       )      ,					args.shard_size       ):
								lowerCamelCase_         =     grouped_dataset[shard : shard + args.shard_size]
								lowerCamelCase_         =     len(dataset_snapshot["input_ids"]       )
								lowerCamelCase_         =     os.path.join(lowerCamelCase__      ,					F'dataset-{shard_count}-{records_containing}.tfrecord'       )
								lowerCamelCase_         =     get_serialized_examples(lowerCamelCase__       )

								with tf.io.TFRecordWriter(lowerCamelCase__       ) as out_file:
												for i in range(len(lowerCamelCase__       )       ):
																lowerCamelCase_         =     serialized_examples[i]
																out_file.write(lowerCamelCase__       )
												print("Wrote file {} containing {} records".format(lowerCamelCase__      ,					lowerCamelCase__       )       )

								shard_count += 1
								total_records += records_containing

				with open(F'split-{args.split}-records-count.txt'      ,					"w"       ) as f:
								print(F'Total {args.split} records: {total_records}'      ,					file=lowerCamelCase__       )


if __name__ == "__main__":
			__A							=parse_args()
			main(args)


