


'''simple docstring'''

import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('''0.8.3'''):
							raise Exception('''requires gluonnlp == 0.8.3''')

if version.parse(mx.__version__) != version.parse('''1.5.0'''):
							raise Exception('''requires mxnet == 1.5.0''')

logging.set_verbosity_info()
lowerCAmelCase :List[Any]             =     logging.get_logger(__name__)

lowerCAmelCase :Dict             =     '''The Nymphenburg Palace is a beautiful palace in Munich!'''





def lowerCamelCase     (     lowerCAmelCase							:      str						,						lowerCAmelCase							:      str						):





	"""simple docstring"""







	__magic_name__			:       str	      =	{
	    'attention_cell': 'multi_head',
	    'num_layers': 4,
	    'units': 1024,
	    'hidden_size': 768,
	    'max_length': 512,
	    'num_heads': 8,
	    'scaled': True,
	    'dropout': 0.1,
	    'use_residual': True,
	    'embed_size': 1024,
	    'embed_dropout': 0.1,
	    'word_embed': None,
	    'layer_norm_eps': 1e-5,
	    'token_type_vocab_size': 2,
	}

	__magic_name__			:       Union[str, Any]	      =	bort_4_8_768_1024_hparams

	# Let's construct the original Bort model here
	# Taken from official BERT implementation, see:
	# https://github.com/alexa/bort/blob/master/bort/bort.py
	__magic_name__			:       int	      =	BERTEncoder(
	    attention_cell=predefined_args['attention_cell']						,						num_layers=predefined_args['num_layers']						,						units=predefined_args['units']						,						hidden_size=predefined_args['hidden_size']						,						max_length=predefined_args['max_length']						,						num_heads=predefined_args['num_heads']						,						scaled=predefined_args['scaled']						,						dropout=predefined_args['dropout']						,						output_attention=__lowerCAmelCase						,						output_all_encodings=__lowerCAmelCase						,						use_residual=predefined_args['use_residual']						,						activation=predefined_args.get('activation'						,						'gelu'						)						,						layer_norm_eps=predefined_args.get('layer_norm_eps'						,						__lowerCAmelCase						)						,						)

	# Vocab information needs to be fetched first
	# It's the same as RoBERTa, so RobertaTokenizer can be used later
	__magic_name__			:       Optional[int]	      =	'openwebtext_ccnews_stories_books_cased'

	# Specify download folder to Gluonnlp's vocab
	__magic_name__			:       int	      =	os.path.join(get_home_dir()						,						'models'						)
	__magic_name__			:       Tuple	      =	_load_vocab(__lowerCAmelCase						,						__lowerCAmelCase						,						__lowerCAmelCase						,						cls=__lowerCAmelCase						)

	__magic_name__			:       str	      =	nlp.model.BERTModel(
	    __lowerCAmelCase						,						len(__lowerCAmelCase						)						,						units=predefined_args['units']						,						embed_size=predefined_args['embed_size']						,						embed_dropout=predefined_args['embed_dropout']						,						word_embed=predefined_args['word_embed']						,						use_pooler=__lowerCAmelCase						,						use_token_type_embed=__lowerCAmelCase						,						token_type_vocab_size=predefined_args['token_type_vocab_size']						,						use_classifier=__lowerCAmelCase						,						use_decoder=__lowerCAmelCase						,						)

	original_bort.load_parameters(__lowerCAmelCase						,						cast_dtype=__lowerCAmelCase						,						ignore_extra=__lowerCAmelCase						)
	__magic_name__			:       Any	      =	original_bort._collect_params_with_prefix()

	# Build our config 🤗
	__magic_name__			:       Optional[Any]	      =	{
	    'architectures': ['BertForMaskedLM'],
	    'attention_probs_dropout_prob': predefined_args['dropout'],
	    'hidden_act': 'gelu',
	    'hidden_dropout_prob': predefined_args['dropout'],
	    'hidden_size': predefined_args['embed_size'],
	    'initializer_range': 0.02,
	    'intermediate_size': predefined_args['hidden_size'],
	    'layer_norm_eps': predefined_args['layer_norm_eps'],
	    'max_position_embeddings': predefined_args['max_length'],
	    'model_type': 'bort',
	    'num_attention_heads': predefined_args['num_heads'],
	    'num_hidden_layers': predefined_args['num_layers'],
	    'pad_token_id': 1,  # 2 = BERT, 1 = RoBERTa
	    'type_vocab_size': 1,  # 2 = BERT, 1 = RoBERTa
	    'vocab_size': len(__lowerCAmelCase						),
	}

	__magic_name__			:       Union[str, Any]	      =	BertConfig.from_dict(__lowerCAmelCase						)
	__magic_name__			:       Union[str, Any]	      =	BertForMaskedLM(__lowerCAmelCase						)
	hf_bort_model.eval()

	# Parameter mapping table (Gluonnlp to Transformers)
	# * denotes layer index
	#
	# | Gluon Parameter                                                | Transformers Parameter
	# | -------------------------------------------------------------- | ----------------------
	# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
	# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
	# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
	# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
	# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
	# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
	# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
	# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
	# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
	# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
	# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
	# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
	# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
	# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
	# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
	# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
	# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
	# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
	# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
	# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

	# Helper function to convert MXNET Arrays to PyTorch
	def to_torch(lowerCAmelCase							:      int						) -> nn.Parameter:
		return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()						)						)

	# Check param shapes and map new HF param back
	def check_and_map_params(lowerCAmelCase							:      int						,						lowerCAmelCase							:      List[str]						):
		__magic_name__			:       Optional[Any]	      =	hf_param.shape

		__magic_name__			:       Union[str, Any]	      =	to_torch(params[gluon_param]						)
		__magic_name__			:       Optional[Any]	      =	gluon_param.shape

		assert (
		    shape_hf == shape_gluon
		), f'The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers'

		return gluon_param

	__magic_name__			:       Dict	      =	check_and_map_params(
	    hf_bort_model.bert.embeddings.word_embeddings.weight						,						'word_embed.0.weight'						)
	__magic_name__			:       Union[str, Any]	      =	check_and_map_params(
	    hf_bort_model.bert.embeddings.position_embeddings.weight						,						'encoder.position_weight'						)
	__magic_name__			:       Union[str, Any]	      =	check_and_map_params(
	    hf_bort_model.bert.embeddings.LayerNorm.bias						,						'encoder.layer_norm.beta'						)
	__magic_name__			:       str	      =	check_and_map_params(
	    hf_bort_model.bert.embeddings.LayerNorm.weight						,						'encoder.layer_norm.gamma'						)

	# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
	__magic_name__			:       Dict	      =	torch.zeros_like(
	    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data						)

	for i in range(hf_bort_config.num_hidden_layers						):
		__magic_name__			:       List[str]	      =	hf_bort_model.bert.encoder.layer[i]

		# self attention
		__magic_name__			:       int	      =	layer.attention.self

		__magic_name__			:       Union[str, Any]	      =	check_and_map_params(
		    self_attn.key.bias.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_key.bias'						)

		__magic_name__			:       Tuple	      =	check_and_map_params(
		    self_attn.key.weight.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_key.weight'						)
		__magic_name__			:       Any	      =	check_and_map_params(
		    self_attn.query.bias.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_query.bias'						)
		__magic_name__			:       List[Any]	      =	check_and_map_params(
		    self_attn.query.weight.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_query.weight'						)
		__magic_name__			:       Optional[Any]	      =	check_and_map_params(
		    self_attn.value.bias.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_value.bias'						)
		__magic_name__			:       Optional[Any]	      =	check_and_map_params(
		    self_attn.value.weight.data						,						f'encoder.transformer_cells.{i}.attention_cell.proj_value.weight'						)

		# self attention output
		__magic_name__			:       int	      =	layer.attention.output

		__magic_name__			:       Optional[int]	      =	check_and_map_params(
		    self_output.dense.bias						,						f'encoder.transformer_cells.{i}.proj.bias'						)
		__magic_name__			:       int	      =	check_and_map_params(
		    self_output.dense.weight						,						f'encoder.transformer_cells.{i}.proj.weight'						)
		__magic_name__			:       Dict	      =	check_and_map_params(
		    self_output.LayerNorm.bias						,						f'encoder.transformer_cells.{i}.layer_norm.beta'						)
		__magic_name__			:       str	      =	check_and_map_params(
		    self_output.LayerNorm.weight						,						f'encoder.transformer_cells.{i}.layer_norm.gamma'						)

		# intermediate
		__magic_name__			:       Tuple	      =	layer.intermediate

		__magic_name__			:       Optional[Any]	      =	check_and_map_params(
		    intermediate.dense.bias						,						f'encoder.transformer_cells.{i}.ffn.ffn_1.bias'						)
		__magic_name__			:       int	      =	check_and_map_params(
		    intermediate.dense.weight						,						f'encoder.transformer_cells.{i}.ffn.ffn_1.weight'						)

		# output
		__magic_name__			:       List[str]	      =	layer.output

		__magic_name__			:       List[Any]	      =	check_and_map_params(
		    bert_output.dense.bias						,						f'encoder.transformer_cells.{i}.ffn.ffn_2.bias'						)
		__magic_name__			:       List[Any]	      =	check_and_map_params(
		    bert_output.dense.weight						,						f'encoder.transformer_cells.{i}.ffn.ffn_2.weight'						)
		__magic_name__			:       List[str]	      =	check_and_map_params(
		    bert_output.LayerNorm.bias						,						f'encoder.transformer_cells.{i}.ffn.layer_norm.beta'						)
		__magic_name__			:       List[str]	      =	check_and_map_params(
		    bert_output.LayerNorm.weight						,						f'encoder.transformer_cells.{i}.ffn.layer_norm.gamma'						)

	# Save space and energy 🎄
	hf_bort_model.half()

	# Compare output of both models
	__magic_name__			:       Optional[Any]	      =	RobertaTokenizer.from_pretrained('roberta-base'						)

	__magic_name__			:       Optional[Any]	      =	tokenizer.encode_plus(__lowerCAmelCase						)['input_ids']

	# Get gluon output
	__magic_name__			:       Any	      =	mx.nd.array([input_ids]						)
	__magic_name__			:       int	      =	original_bort(inputs=__lowerCAmelCase						,						token_types=[]						)

	# Get Transformer output (save and reload model again)
	hf_bort_model.save_pretrained(__lowerCAmelCase						)
	__magic_name__			:       List[str]	      =	BertModel.from_pretrained(__lowerCAmelCase						)
	hf_bort_model.eval()

	__magic_name__			:       Optional[int]	      =	tokenizer.encode_plus(__lowerCAmelCase						,						return_tensors='pt'						)
	__magic_name__			:       Optional[int]	      =	hf_bort_model(**__lowerCAmelCase						)[0]

	__magic_name__			:       Union[str, Any]	      =	output_gluon[0].asnumpy()
	__magic_name__			:       Optional[Any]	      =	output_hf[0].detach().numpy()

	__magic_name__			:       Any	      =	np.max(np.abs(hf_layer - gluon_layer						)						).item()
	__magic_name__			:       int	      =	np.allclose(__lowerCAmelCase						,						__lowerCAmelCase						,						atol=1e-3						)

	if success:
		print('✔️ Both model do output the same tensors'						)
	else:
		print('❌ Both model do **NOT** output the same tensors'						)
		print('Absolute difference is:'						,						__lowerCAmelCase						)


if __name__ == "__main__":
							lowerCAmelCase :str             =     argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--bort_checkpoint_path''', default=None, type=str, required=True, help='''Path the official Bort params file.'''
							)
							parser.add_argument(
							    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
							)
							lowerCAmelCase :Optional[Any]             =     parser.parse_args()
							convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)

import inspect
import unittest

from transformers import ViTMSNConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch
			from torch import nn

			from transformers import ViTMSNForImageClassification, ViTMSNModel
			from transformers.models.vit_msn.modeling_vit_msn import VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
			from PIL import Image

			from transformers import ViTImageProcessor





class       _lowerCAmelCase   :




				def __init__(   self							,							_UpperCamelCase							,							_UpperCamelCase=13							,							_UpperCamelCase=30							,							_UpperCamelCase=2							,							_UpperCamelCase=3							,							_UpperCamelCase=True							,							_UpperCamelCase=True							,							_UpperCamelCase=32							,							_UpperCamelCase=5							,							_UpperCamelCase=4							,							_UpperCamelCase=37							,							_UpperCamelCase="gelu"							,							_UpperCamelCase=0.1							,							_UpperCamelCase=0.1							,							_UpperCamelCase=10							,							_UpperCamelCase=0.02							,							_UpperCamelCase=None							,							)							-> List[Any]:
							lowerCAmelCase_         =      parent
							lowerCAmelCase_         =      batch_size
							lowerCAmelCase_         =      image_size
							lowerCAmelCase_         =      patch_size
							lowerCAmelCase_         =      num_channels
							lowerCAmelCase_         =      is_training
							lowerCAmelCase_         =      use_labels
							lowerCAmelCase_         =      hidden_size
							lowerCAmelCase_         =      num_hidden_layers
							lowerCAmelCase_         =      num_attention_heads
							lowerCAmelCase_         =      intermediate_size
							lowerCAmelCase_         =      hidden_act
							lowerCAmelCase_         =      hidden_dropout_prob
							lowerCAmelCase_         =      attention_probs_dropout_prob
							lowerCAmelCase_         =      type_sequence_label_size
							lowerCAmelCase_         =      initializer_range
							lowerCAmelCase_         =      scope

							# in ViT MSN, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
							lowerCAmelCase_         =      (image_size // patch_size) ** 2
							lowerCAmelCase_         =      num_patches + 1




				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

							lowerCAmelCase_         =      None
							if self.use_labels:
										lowerCAmelCase_         =      ids_tensor([self.batch_size]							,							self.type_sequence_label_size	)

							lowerCAmelCase_         =      self.get_config()

							return config, pixel_values, labels




				def 	__a		(   self	)							-> Optional[Any]:
							return ViTMSNConfig(
							    image_size=self.image_size							,							patch_size=self.patch_size							,							num_channels=self.num_channels							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							initializer_range=self.initializer_range							,							)




				def 	__a		(   self							,							_UpperCamelCase							,							_UpperCamelCase							,							_UpperCamelCase	)							-> str:
							lowerCAmelCase_         =      ViTMSNModel(config=_UpperCamelCase	)
							model.to(_UpperCamelCase	)
							model.eval()
							lowerCAmelCase_         =      model(_UpperCamelCase	)
							self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)	)




				def 	__a		(   self							,							_UpperCamelCase							,							_UpperCamelCase							,							_UpperCamelCase	)							-> str:
							lowerCAmelCase_         =      self.type_sequence_label_size
							lowerCAmelCase_         =      ViTMSNForImageClassification(_UpperCamelCase	)
							model.to(_UpperCamelCase	)
							model.eval()
							lowerCAmelCase_         =      model(_UpperCamelCase							,							labels=_UpperCamelCase	)
							print("Pixel and labels shape: {pixel_values.shape}, {labels.shape}"	)
							print("Labels: {labels}"	)
							self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.type_sequence_label_size)	)

							# test greyscale images
							lowerCAmelCase_         =      1
							lowerCAmelCase_         =      ViTMSNForImageClassification(_UpperCamelCase	)
							model.to(_UpperCamelCase	)
							model.eval()

							lowerCAmelCase_         =      floats_tensor([self.batch_size, 1, self.image_size, self.image_size]	)
							lowerCAmelCase_         =      model(_UpperCamelCase	)
							self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.type_sequence_label_size)	)




				def 	__a		(   self	)							-> Union[str, Any]:
							lowerCAmelCase_         =      self.prepare_config_and_inputs()
							lowerCAmelCase_      ,    lowerCAmelCase_      ,    lowerCAmelCase_         =      config_and_inputs
							lowerCAmelCase_         =      {"pixel_values": pixel_values}
							return config, inputs_dict





@require_torch
class       _lowerCAmelCase   (      __a   , __a   , unittest.TestCase			):
				_lowercase     						=(ViTMSNModel, ViTMSNForImageClassification) if is_torch_available() else ()
				_lowercase     						=(
				    {'''feature-extraction''': ViTMSNModel, '''image-classification''': ViTMSNForImageClassification}
				    if is_torch_available()
				    else {}
				)

				_lowercase     						=False
				_lowercase     						=False
				_lowercase     						=False
				_lowercase     						=False




				def 	__a		(   self	)							-> Optional[int]:
							lowerCAmelCase_         =      ViTMSNModelTester(self	)
							lowerCAmelCase_         =      ConfigTester(self							,							config_class=_UpperCamelCase							,							has_text_modality=_UpperCamelCase							,							hidden_size=37	)




				def 	__a		(   self	)							-> Optional[int]:
							self.config_tester.run_common_tests()




				@unittest.skip(reason="ViTMSN does not use inputs_embeds"	)
				def 	__a		(   self	)							-> str:
							pass




				def 	__a		(   self	)							-> Any:
							lowerCAmelCase_      ,    lowerCAmelCase_         =      self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
										lowerCAmelCase_         =      model_class(_UpperCamelCase	)
										self.assertIsInstance(model.get_input_embeddings()							,							(nn.Module)	)
										lowerCAmelCase_         =      model.get_output_embeddings()
										self.assertTrue(x is None or isinstance(_UpperCamelCase							,							nn.Linear	)	)




				def 	__a		(   self	)							-> Optional[Any]:
							lowerCAmelCase_      ,    lowerCAmelCase_         =      self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
										lowerCAmelCase_         =      model_class(_UpperCamelCase	)
										lowerCAmelCase_         =      inspect.signature(model.forward	)
										# signature.parameters is an OrderedDict => so arg_names order is deterministic
										lowerCAmelCase_         =      [*signature.parameters.keys()]

										lowerCAmelCase_         =      ["pixel_values"]
										self.assertListEqual(arg_names[:1]							,							_UpperCamelCase	)




				def 	__a		(   self	)							-> str:
							lowerCAmelCase_         =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_UpperCamelCase	)




				def 	__a		(   self	)							-> Any:
							lowerCAmelCase_         =      self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*_UpperCamelCase	)




				@slow
				def 	__a		(   self	)							-> Union[str, Any]:
							for model_name in VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowerCAmelCase_         =      ViTMSNModel.from_pretrained(_UpperCamelCase	)
										self.assertIsNotNone(_UpperCamelCase	)






def   lowerCamelCase__							(							):




			"""simple docstring"""


			lowerCAmelCase_         =      Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"							)
			return image





@require_torch
@require_vision
class       _lowerCAmelCase   (      unittest.TestCase			):




				@cached_property
				def 	__a		(   self	)							-> List[str]:
							return ViTImageProcessor.from_pretrained("facebook/vit-msn-small"	) if is_vision_available() else None




				@slow
				def 	__a		(   self	)							-> int:
							torch.manual_seed(2	)
							lowerCAmelCase_         =      ViTMSNForImageClassification.from_pretrained("facebook/vit-msn-small"	).to(_UpperCamelCase	)

							lowerCAmelCase_         =      self.default_image_processor
							lowerCAmelCase_         =      prepare_img()
							lowerCAmelCase_         =      image_processor(images=_UpperCamelCase							,							return_tensors="pt"	).to(_UpperCamelCase	)

							# forward pass
							with torch.no_grad():
										lowerCAmelCase_         =      model(**_UpperCamelCase	)

							# verify the logits
							lowerCAmelCase_         =      torch.Size((1, 1_000)	)
							self.assertEqual(outputs.logits.shape							,							_UpperCamelCase	)

							lowerCAmelCase_         =      torch.tensor([-0.0803, -0.4454, -0.2375]	).to(_UpperCamelCase	)

							self.assertTrue(torch.allclose(outputs.logits[0, :3]							,							_UpperCamelCase							,							atol=1e-4	)	)



