




import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
					UpperCamelCase__					    =    argparse.ArgumentParser(
					    description=(
					        """Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"""
					        """ Distillation"""
					    )
					)
					parser.add_argument("""--model_type""", default="""bert""", choices=["""bert"""])
					parser.add_argument("""--model_name""", default="""bert-base-uncased""", type=str)
					parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_bert-base-uncased_0247911.pth""", type=str)
					parser.add_argument("""--vocab_transform""", action="""store_true""")
					UpperCamelCase__					    =    parser.parse_args()

					if args.model_type == "bert":
										UpperCamelCase__					    =    BertForMaskedLM.from_pretrained(args.model_name)
										UpperCamelCase__					    =    """bert"""
					else:
										raise ValueError("""args.model_type should be \"bert\".""")

					UpperCamelCase__					    =    model.state_dict()
					UpperCamelCase__					    =    {}

					for w in ["word_embeddings", "position_embeddings"]:
										UpperCamelCase__					    =    state_dict[f'''{prefix}.embeddings.{w}.weight''']
					for w in ["weight", "bias"]:
										UpperCamelCase__					    =    state_dict[f'''{prefix}.embeddings.LayerNorm.{w}''']

					UpperCamelCase__					    =    0
					for teacher_idx in [0, 2, 4, 7, 9, 11]:
										for w in ["weight", "bias"]:
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
															]
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
															]
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
															]

															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
															]
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
															]

															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
															]
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
															]
															UpperCamelCase__					    =    state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
															]
										std_idx += 1

					UpperCamelCase__					    =    state_dict["""cls.predictions.decoder.weight"""]
					UpperCamelCase__					    =    state_dict["""cls.predictions.bias"""]
					if args.vocab_transform:
										for w in ["weight", "bias"]:
															UpperCamelCase__					    =    state_dict[f'''cls.predictions.transform.dense.{w}''']
															UpperCamelCase__					    =    state_dict[f'''cls.predictions.transform.LayerNorm.{w}''']

					print(f'''N layers selected for distillation: {std_idx}''')
					print(f'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

					print(f'''Save transferred checkpoint to {args.dump_checkpoint}.''')
					torch.save(compressed_sd, args.dump_checkpoint)










def 			_a   (		SCREAMING_SNAKE_CASE_							:       int  ):
					if divisor % 5 == 0 or divisor % 2 == 0:
										return 0
					__lowerCAmelCase            = 1
					__lowerCAmelCase            = 1
					while repunit:
										__lowerCAmelCase            = (10 * repunit + 1) % divisor
										repunit_index += 1
					return repunit_index







def 			_a   (		SCREAMING_SNAKE_CASE_							:       int = 1_00_00_00  ):
					__lowerCAmelCase            = limit - 1
					if divisor % 2 == 0:
										divisor += 1
					while least_divisible_repunit(SCREAMING_SNAKE_CASE_  ) <= limit:
										divisor += 2
					return divisor


if __name__ == "__main__":
					print(f'''{solution() = }''')




