



"""simple docstring"""
import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase						:		Dict			= logging.get_logger(__name__)


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase ,  __lowerCAmelCase							) ->						Union[str, Any]:







  '''simple docstring'''


  lowercase_								= os.path.abspath(_lowerCamelCase							)
  logger.info(F'''Converting TensorFlow checkpoint from {tf_path}'''							)
  # Load weights from TF model
  lowercase_								= tf.train.list_variables(_lowerCamelCase							)
  lowercase_								= []
  lowercase_								= []
  lowercase_								= []
  for full_name, shape in init_vars:
    # logger.info(f"Loading TF weight {name} with shape {shape}")
    lowercase_								= full_name.split("""/"""							)
    if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
      logger.info(F'''Skipping non-model layer {full_name}'''							)
      continue
    if "optimizer" in full_name:
      logger.info(F'''Skipping optimization layer {full_name}'''							)
      continue
    if name[0] == "model":
      # ignore initial 'model'
      lowercase_								= name[1:]
    # figure out how many levels deep the name is
    lowercase_								= 0
    for _name in name:
      if _name.startswith("""layer_with_weights"""							):
        depth += 1
      else:
        break
    layer_depth.append(_lowerCamelCase							)
    # read data
    lowercase_								= tf.train.load_variable(_lowerCamelCase ,  _lowerCamelCase							)
    names.append("""/""".join(_lowerCamelCase							)							)
    arrays.append(_lowerCamelCase							)
  logger.info(F'''Read a total of {len(_lowerCamelCase							):,} layers'''							)

  # Sanity check
  if len(set(_lowerCamelCase							)							) != 1:
    raise ValueError(F'''Found layer names with different depths (layer depth {list(set(_lowerCamelCase							)							)})'''							)
  lowercase_								= list(set(_lowerCamelCase							)							)[0]
  if layer_depth != 1:
    raise ValueError(
        """The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"""
        """ heads."""							)

  # convert layers
  logger.info("""Converting weights..."""							)
  for full_name, array in zip(_lowerCamelCase ,  _lowerCamelCase							):
    lowercase_								= full_name.split("""/"""							)
    lowercase_								= model
    lowercase_								= []
    for i, m_name in enumerate(_lowerCamelCase							):
      if m_name == ".ATTRIBUTES":
        # variable names end with .ATTRIBUTES/VARIABLE_VALUE
        break
      if m_name.startswith("""layer_with_weights"""							):
        lowercase_								= int(m_name.split("""-"""							)[-1]							)
        if layer_num <= 2:
          # embedding layers
          # layer_num 0: word_embeddings
          # layer_num 1: position_embeddings
          # layer_num 2: token_type_embeddings
          continue
        elif layer_num == 3:
          # embedding LayerNorm
          trace.extend(["""embeddings""", """LayerNorm"""]							)
          lowercase_								= getattr(_lowerCamelCase ,  """embeddings"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """LayerNorm"""							)
        elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
          # encoder layers
          trace.extend(["""encoder""", """layer""", str(layer_num - 4							)]							)
          lowercase_								= getattr(_lowerCamelCase ,  """encoder"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """layer"""							)
          lowercase_								= pointer[layer_num - 4]
        elif layer_num == config.num_hidden_layers + 4:
          # pooler layer
          trace.extend(["""pooler""", """dense"""]							)
          lowercase_								= getattr(_lowerCamelCase ,  """pooler"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """dense"""							)
      elif m_name == "embeddings":
        trace.append("""embeddings"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """embeddings"""							)
        if layer_num == 0:
          trace.append("""word_embeddings"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """word_embeddings"""							)
        elif layer_num == 1:
          trace.append("""position_embeddings"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """position_embeddings"""							)
        elif layer_num == 2:
          trace.append("""token_type_embeddings"""							)
          lowercase_								= getattr(_lowerCamelCase ,  """token_type_embeddings"""							)
        else:
          raise ValueError(F'''Unknown embedding layer with name {full_name}'''							)
        trace.append("""weight"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """weight"""							)
      elif m_name == "_attention_layer":
        # self-attention layer
        trace.extend(["""attention""", """self"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """attention"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """self"""							)
      elif m_name == "_attention_layer_norm":
        # output attention norm
        trace.extend(["""attention""", """output""", """LayerNorm"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """attention"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """output"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """LayerNorm"""							)
      elif m_name == "_attention_output_dense":
        # output attention dense
        trace.extend(["""attention""", """output""", """dense"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """attention"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """output"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """dense"""							)
      elif m_name == "_output_dense":
        # output dense
        trace.extend(["""output""", """dense"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """output"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """dense"""							)
      elif m_name == "_output_layer_norm":
        # output dense
        trace.extend(["""output""", """LayerNorm"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """output"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """LayerNorm"""							)
      elif m_name == "_key_dense":
        # attention key
        trace.append("""key"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """key"""							)
      elif m_name == "_query_dense":
        # attention query
        trace.append("""query"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """query"""							)
      elif m_name == "_value_dense":
        # attention value
        trace.append("""value"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """value"""							)
      elif m_name == "_intermediate_dense":
        # attention intermediate dense
        trace.extend(["""intermediate""", """dense"""]							)
        lowercase_								= getattr(_lowerCamelCase ,  """intermediate"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """dense"""							)
      elif m_name == "_output_layer_norm":
        # output layer norm
        trace.append("""output"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """output"""							)
      # weights & biases
      elif m_name in ["bias", "beta"]:
        trace.append("""bias"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """bias"""							)
      elif m_name in ["kernel", "gamma"]:
        trace.append("""weight"""							)
        lowercase_								= getattr(_lowerCamelCase ,  """weight"""							)
      else:
        logger.warning(F'''Ignored {m_name}'''							)
        # for certain layers reshape is necessary
    lowercase_								= ".".join(_lowerCamelCase							)
    if re.match(R"""(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)""" ,  _lowerCamelCase							) or re.match(
        R"""(\S+)\.attention\.output\.dense\.weight""" ,  _lowerCamelCase							):
      lowercase_								= array.reshape(pointer.data.shape							)
    if "kernel" in full_name:
      lowercase_								= array.transpose()
    if pointer.shape == array.shape:
      lowercase_								= torch.from_numpy(_lowerCamelCase							)
    else:
      raise ValueError(
          F'''Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'''
          F''' {array.shape}'''							)
    logger.info(F'''Successfully set variable {full_name} to PyTorch layer {trace}'''							)
  return model


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase ,  __lowerCAmelCase							) ->						List[str]:







  '''simple docstring'''


  logger.info(F'''Loading model based on config from {config_path}...'''							)
  lowercase_								= BertConfig.from_json_file(_lowerCamelCase							)
  lowercase_								= BertModel(_lowerCamelCase							)

  # Load weights from checkpoint
  logger.info(F'''Loading weights from checkpoint {tf_checkpoint_path}...'''							)
  load_tfa_weights_in_bert(_lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase							)

  # Save pytorch-model
  logger.info(F'''Saving PyTorch model to {pytorch_dump_path}...'''							)
  torch.save(model.state_dict() ,  _lowerCamelCase							)


if __name__ == "__main__":
    UpperCAmelCase						:		Optional[Any]			= argparse.ArgumentParser()
    parser.add_argument(
        "--tf_checkpoint_path", type=str, required=True, help="Path to the TensorFlow 2.x checkpoint path."
    )
    parser.add_argument(
        "--bert_config_file",
        type=str,
        required=True,
        help="The config json file corresponding to the BERT model. This specifies the model architecture.",
    )
    parser.add_argument(
        "--pytorch_dump_path",
        type=str,
        required=True,
        help="Path to the output PyTorch model (must include filename).",
    )
    UpperCAmelCase						:		List[str]			= parser.parse_args()
    convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)






"""simple docstring"""
def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase							) ->						int:







  '''simple docstring'''


  while a != 0:
    lowercase_					,    lowercase_								= b % a, a
  return b


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase							) ->						int:







  '''simple docstring'''


  if gcd(__lowerCAmelCase ,  __lowerCAmelCase							) != 1:
    lowercase_								= F'''mod inverse of {a!r} and {m!r} does not exist'''
    raise ValueError(__lowerCAmelCase							)
  lowercase_					,    lowercase_					,    lowercase_								= 1, 0, a
  lowercase_					,    lowercase_					,    lowercase_								= 0, 1, m
  while va != 0:
    lowercase_								= ua // va
    lowercase_					,    lowercase_					,    lowercase_					,    lowercase_					,    lowercase_					,    lowercase_								= (ua - q * va), (ua - q * va), (ua - q * va), va, va, va
  return ua % m

