
"""simple docstring"""






import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
					UpperCAmelCase        =				argparse.ArgumentParser(
					    description=(
					        """Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"""
					        """ Distillation"""
					    )
					)
					parser.add_argument("""--model_type""", default="""bert""", choices=["""bert"""])
					parser.add_argument("""--model_name""", default="""bert-base-uncased""", type=str)
					parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_bert-base-uncased_0247911.pth""", type=str)
					parser.add_argument("""--vocab_transform""", action="""store_true""")
					UpperCAmelCase        =				parser.parse_args()

					if args.model_type == "bert":
										UpperCAmelCase        =				BertForMaskedLM.from_pretrained(args.model_name)
										UpperCAmelCase        =				"""bert"""
					else:
										raise ValueError("""args.model_type should be \"bert\".""")

					UpperCAmelCase        =				model.state_dict()
					UpperCAmelCase        =				{}

					for w in ["word_embeddings", "position_embeddings"]:
										UpperCAmelCase        =				state_dict[f"""{prefix}.embeddings.{w}.weight"""]
					for w in ["weight", "bias"]:
										UpperCAmelCase        =				state_dict[f"""{prefix}.embeddings.LayerNorm.{w}"""]

					UpperCAmelCase        =				0
					for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
										for w in ["weight", "bias"]:
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
															]
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
															]
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
															]

															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
															]
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
															]

															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
															]
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
															]
															UpperCAmelCase        =				state_dict[
															    f"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
															]
										std_idx += 1

					UpperCAmelCase        =				state_dict["""cls.predictions.decoder.weight"""]
					UpperCAmelCase        =				state_dict["""cls.predictions.bias"""]
					if args.vocab_transform:
										for w in ["weight", "bias"]:
															UpperCAmelCase        =				state_dict[f"""cls.predictions.transform.dense.{w}"""]
															UpperCAmelCase        =				state_dict[f"""cls.predictions.transform.LayerNorm.{w}"""]

					print(f"""N layers selected for distillation: {std_idx}""")
					print(f"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

					print(f"""Save transferred checkpoint to {args.dump_checkpoint}.""")
					torch.save(compressed_sd, args.dump_checkpoint)




"""simple docstring"""






def __magic_name__	(				_lowerCamelCase:		Tuple,					_lowerCamelCase:		Any			)					->						Optional[Any]:

	'''simple docstring'''




	lowerCAmelCase												=  [1]
	for i in range(2,					_lowerCamelCase			):
		factorials.append(factorials[-1] * i			)
	assert 0 <= k < factorials[-1] * n, "k out of bounds"

	lowerCAmelCase												=  []
	lowerCAmelCase												=  list(range(_lowerCamelCase			)			)

	# Find permutation
	while factorials:
		lowerCAmelCase												=  factorials.pop()
		lowerCAmelCase							,       lowerCAmelCase												=  divmod(_lowerCamelCase,					_lowerCamelCase			)
		permutation.append(elements[number]			)
		elements.remove(elements[number]			)
	permutation.append(elements[0]			)

	return permutation


if __name__ == "__main__":
					import doctest

					doctest.testmod()


