





"""simple docstring"""




import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
					import torch

					from transformers import AutoModelForImageClassification

if is_vision_available():
					from transformers import AutoImageProcessor


@require_torch
@require_vision
class   A      (unittest.TestCase   ):
			'''simple docstring'''




			@slow
			def        a_	(      self       :   List[Any]       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       AutoImageProcessor.from_pretrained("""microsoft/dit-base-finetuned-rvlcdip"""       )
					A__  				=       AutoModelForImageClassification.from_pretrained("""microsoft/dit-base-finetuned-rvlcdip"""       )
					model.to(__lowerCAmelCase       )

					from datasets import load_dataset

					A__  				=       load_dataset("""nielsr/rvlcdip-demo"""       )

					A__  				=       dataset["""train"""][0]["""image"""].convert("""RGB"""       )

					A__  				=       image_processor(__lowerCAmelCase							,     return_tensors="""pt"""       ).to(__lowerCAmelCase       )

					# forward pass
					with torch.no_grad():
							A__  				=       model(**__lowerCAmelCase       )
							A__  				=       outputs.logits

					A__  				=       torch.Size((1, 16)       )
					self.assertEqual(logits.shape							,     __lowerCAmelCase       )

					A__  				=       torch.tensor(
					    [-0.4_1_5_8, -0.4_0_9_2, -0.4_3_4_7]							,     device=__lowerCAmelCase							,     dtype=torch.float							,     )
					self.assertTrue(torch.allclose(logits[0, :3]							,     __lowerCAmelCase							,     atol=1e-4       )       )










import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
					A   : List[str]			=				argparse.ArgumentParser(
					    description=(
					        '''Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'''
					        ''' Distillation'''
					    )
					)
					parser.add_argument('''--model_type''', default='''bert''', choices=['''bert'''])
					parser.add_argument('''--model_name''', default='''bert-base-uncased''', type=str)
					parser.add_argument('''--dump_checkpoint''', default='''serialization_dir/tf_bert-base-uncased_0247911.pth''', type=str)
					parser.add_argument('''--vocab_transform''', action='''store_true''')
					A   : Tuple			=				parser.parse_args()

					if args.model_type == "bert":
										A   : Dict			=				BertForMaskedLM.from_pretrained(args.model_name)
										A   : List[str]			=				'''bert'''
					else:
										raise ValueError('''args.model_type should be "bert".''')

					A   : Optional[Any]			=				model.state_dict()
					A   : int			=				{}

					for w in ["word_embeddings", "position_embeddings"]:
										A   : str			=				state_dict[F'''{prefix}.embeddings.{w}.weight''']
					for w in ["weight", "bias"]:
										A   : Any			=				state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

					A   : Tuple			=				0
					for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
										for w in ["weight", "bias"]:
															A   : Optional[Any]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
															]
															A   : Optional[Any]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
															]
															A   : Optional[Any]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
															]

															A   : int			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
															]
															A   : List[Any]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
															]

															A   : List[str]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
															]
															A   : Union[str, Any]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
															]
															A   : List[str]			=				state_dict[
															    F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
															]
										std_idx += 1

					A   : int			=				state_dict['''cls.predictions.decoder.weight''']
					A   : str			=				state_dict['''cls.predictions.bias''']
					if args.vocab_transform:
										for w in ["weight", "bias"]:
															A   : List[Any]			=				state_dict[F'''cls.predictions.transform.dense.{w}''']
															A   : List[str]			=				state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

					print(F'''N layers selected for distillation: {std_idx}''')
					print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

					print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
					torch.save(compressed_sd, args.dump_checkpoint)



