
"""simple docstring"""




import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
  A__ :       Tuple				= argparse.ArgumentParser(
      description=(
          'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
          ' Distillation'
      )
  )
  parser.add_argument('--model_type', default='bert', choices=['bert'])
  parser.add_argument('--model_name', default='bert-base-uncased', type=str)
  parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
  parser.add_argument('--vocab_transform', action='store_true')
  A__ :       Union[str, Any]				= parser.parse_args()

  if args.model_type == "bert":
    A__ :       Union[str, Any]				= BertForMaskedLM.from_pretrained(args.model_name)
    A__ :       Optional[int]				= 'bert'
  else:
    raise ValueError('args.model_type should be "bert".')

  A__ :       Optional[int]				= model.state_dict()
  A__ :       int				= {}

  for w in ["word_embeddings", "position_embeddings"]:
    A__ :       int				= state_dict[f"""{prefix}.embeddings.{w}.weight"""]
  for w in ["weight", "bias"]:
    A__ :       Any				= state_dict[f"""{prefix}.embeddings.LayerNorm.{w}"""]

  A__ :       Tuple				= 0
  for teacher_idx in [0, 2, 4, 7, 9, 1_1]:
    for w in ["weight", "bias"]:
      A__ :       Optional[Any]				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
      ]
      A__ :       int				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
      ]
      A__ :       Tuple				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
      ]

      A__ :       str				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
      ]
      A__ :       Optional[int]				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
      ]

      A__ :       Optional[int]				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
      ]
      A__ :       Optional[Any]				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
      ]
      A__ :       Optional[Any]				= state_dict[
          f"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
      ]
    std_idx += 1

  A__ :       int				= state_dict['cls.predictions.decoder.weight']
  A__ :       Optional[Any]				= state_dict['cls.predictions.bias']
  if args.vocab_transform:
    for w in ["weight", "bias"]:
      A__ :       int				= state_dict[f"""cls.predictions.transform.dense.{w}"""]
      A__ :       int				= state_dict[f"""cls.predictions.transform.LayerNorm.{w}"""]

  print(f"""N layers selected for distillation: {std_idx}""")
  print(f"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

  print(f"""Save transferred checkpoint to {args.dump_checkpoint}.""")
  torch.save(compressed_sd, args.dump_checkpoint)


"""simple docstring"""




import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()




def 			_lowerCAmelCase		(		_UpperCamelCase			,					_UpperCamelCase			,					_UpperCamelCase			,					_UpperCamelCase    ):






  """simple docstring"""



  _lowercase:      int   			=  BigBirdConfig.from_json_file(_UpperCamelCase    )
  print(f'''Building PyTorch model from configuration: {config}'''    )

  if is_trivia_qa:
    _lowercase:      Union[str, Any]   			=  BigBirdForQuestionAnswering(_UpperCamelCase    )
  else:
    _lowercase:      Dict   			=  BigBirdForPreTraining(_UpperCamelCase    )

  # Load weights from tf checkpoint
  load_tf_weights_in_big_bird(_UpperCamelCase			,					_UpperCamelCase			,					is_trivia_qa=_UpperCamelCase    )

  # Save pytorch-model
  print(f'''Save PyTorch model to {pytorch_dump_path}'''    )
  model.save_pretrained(_UpperCamelCase    )


if __name__ == "__main__":
  A__ :       List[Any]				= argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
  )
  parser.add_argument(
      '--big_bird_config_file',
      default=None,
      type=str,
      required=True,
      help=(
          'The config json file corresponding to the pre-trained BERT model. \n'
          'This specifies the model architecture.'
      ),
  )
  parser.add_argument(
      '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
  )
  parser.add_argument(
      '--is_trivia_qa', action='store_true', help='Whether to convert a model with a trivia_qa head.'
  )
  A__ :       Optional[Any]				= parser.parse_args()
  convert_tf_checkpoint_to_pytorch(
      args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
  )
