

import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
		lowerCamelCase      				=     argparse.ArgumentParser(
		    description=(
		        '''Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'''
		        ''' Distillation'''
		    )
		)
		parser.add_argument('''--model_type''', default='''bert''', choices=['''bert'''])
		parser.add_argument('''--model_name''', default='''bert-base-uncased''', type=str)
		parser.add_argument('''--dump_checkpoint''', default='''serialization_dir/tf_bert-base-uncased_0247911.pth''', type=str)
		parser.add_argument('''--vocab_transform''', action='''store_true''')
		lowerCamelCase      				=     parser.parse_args()

		if args.model_type == "bert":
				lowerCamelCase      				=     BertForMaskedLM.from_pretrained(args.model_name)
				lowerCamelCase      				=     '''bert'''
		else:
				raise ValueError('''args.model_type should be "bert".''')

		lowerCamelCase      				=     model.state_dict()
		lowerCamelCase      				=     {}

		for w in ["word_embeddings", "position_embeddings"]:
				lowerCamelCase      				=     state_dict[F'''{prefix}.embeddings.{w}.weight''']
		for w in ["weight", "bias"]:
				lowerCamelCase      				=     state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

		lowerCamelCase      				=     0
		for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
				for w in ["weight", "bias"]:
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
						]
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
						]
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
						]

						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
						]
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
						]

						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
						]
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
						]
						lowerCamelCase      				=     state_dict[
						    F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
						]
				std_idx += 1

		lowerCamelCase      				=     state_dict['''cls.predictions.decoder.weight''']
		lowerCamelCase      				=     state_dict['''cls.predictions.bias''']
		if args.vocab_transform:
				for w in ["weight", "bias"]:
						lowerCamelCase      				=     state_dict[F'''cls.predictions.transform.dense.{w}''']
						lowerCamelCase      				=     state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

		print(F'''N layers selected for distillation: {std_idx}''')
		print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

		print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
		torch.save(compressed_sd, args.dump_checkpoint)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase      				=     {
    '''configuration_lilt''': ['''LILT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''LiltConfig'''],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase      				=     [
		    '''LILT_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LiltForQuestionAnswering''',
		    '''LiltForSequenceClassification''',
		    '''LiltForTokenClassification''',
		    '''LiltModel''',
		    '''LiltPreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_lilt import LILT_PRETRAINED_CONFIG_ARCHIVE_MAP, LiltConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_lilt import (
				    LILT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LiltForQuestionAnswering,
				    LiltForSequenceClassification,
				    LiltForTokenClassification,
				    LiltModel,
				    LiltPreTrainedModel,
				)

else:
		import sys

		lowerCamelCase      				=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






