



import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
     from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
     import torch_xla.core.xla_model as xm






def     __a					(		__UpperCAmelCase       ):
 if is_torch_version('''<'''							,  '''2.0.0'''       ) or not hasattr(__UpperCAmelCase							,  '''_dynamo'''       ):
  return False
 return isinstance(__UpperCAmelCase							,  torch._dynamo.eval_frame.OptimizedModule       )






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase = True       ):
 a__												= (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

 a__												= is_compiled_module(__UpperCAmelCase       )
 if is_compiled:
  a__												= model
  a__												= model._orig_mod

 if is_deepspeed_available():
  options += (DeepSpeedEngine,)

 while isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ):
  a__												= model.module

 if not keep_fpaa_wrapper:
  a__												= getattr(__UpperCAmelCase							,  '''forward'''       )
  a__												= model.__dict__.pop('''_original_forward'''							,  __UpperCAmelCase       )
  if original_forward is not None:
   while hasattr(__UpperCAmelCase							,  '''__wrapped__'''       ):
    a__												= forward.__wrapped__
    if forward == original_forward:
     break
   a__												= forward
  if getattr(__UpperCAmelCase							,  '''_converted_to_transformer_engine'''							,  __UpperCAmelCase       ):
   convert_model(__UpperCAmelCase							,  to_transformer_engine=__UpperCAmelCase       )

 if is_compiled:
  a__												= model
  a__												= compiled_model

 return model






def     __a					(		):
 PartialState().wait_for_everyone()






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase       ):
 if PartialState().distributed_type == DistributedType.TPU:
  xm.save(__UpperCAmelCase							,  __UpperCAmelCase       )
 elif PartialState().local_process_index == 0:
  torch.save(__UpperCAmelCase							,  __UpperCAmelCase       )






@contextmanager
def     __a					(		**__UpperCAmelCase       ):
 for key, value in kwargs.items():
  a__												= str(__UpperCAmelCase       )

 yield

 for key in kwargs:
  if key.upper() in os.environ:
   del os.environ[key.upper()]






def     __a					(		__UpperCAmelCase       ):
 if not hasattr(__UpperCAmelCase							,  '''__qualname__'''       ) and not hasattr(__UpperCAmelCase							,  '''__name__'''       ):
  a__												= getattr(__UpperCAmelCase							,  '''__class__'''							,  __UpperCAmelCase       )
 if hasattr(__UpperCAmelCase							,  '''__qualname__'''       ):
  return obj.__qualname__
 if hasattr(__UpperCAmelCase							,  '''__name__'''       ):
  return obj.__name__
 return str(__UpperCAmelCase       )






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase       ):
 for key, value in source.items():
  if isinstance(__UpperCAmelCase							,  __UpperCAmelCase       ):
   a__												= destination.setdefault(__UpperCAmelCase							,  {}       )
   merge_dicts(__UpperCAmelCase							,  __UpperCAmelCase       )
  else:
   a__												= value

 return destination






def     __a					(		__UpperCAmelCase = None       ):
 if port is None:
  a__												= 2_9500
 with socket.socket(socket.AF_INET							,  socket.SOCK_STREAM       ) as s:
  return s.connect_ex(('''localhost''', port)       ) == 0










import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
 a__												= ('''dense.weight''', '''attention.self.query''', '''attention.self.key''', '''attention.self.value''')

 a__												= (
     ('''layer.''', '''layer_'''),
     ('''word_embeddings.weight''', '''word_embeddings'''),
     ('''position_embeddings.weight''', '''position_embeddings'''),
     ('''token_type_embeddings.weight''', '''token_type_embeddings'''),
     ('''.''', '''/'''),
     ('''LayerNorm/weight''', '''LayerNorm/gamma'''),
     ('''LayerNorm/bias''', '''LayerNorm/beta'''),
     ('''weight''', '''kernel'''),
 )

 if not os.path.isdir(__UpperCAmelCase       ):
  os.makedirs(__UpperCAmelCase       )

 a__												= model.state_dict()

 def to_tf_var_name(__UpperCAmelCase       ):
  for patt, repl in iter(__UpperCAmelCase       ):
   a__												= name.replace(__UpperCAmelCase							,  __UpperCAmelCase       )
  return f"bert/{name}"

 def create_tf_var(__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
  a__												= tf.dtypes.as_dtype(tensor.dtype       )
  a__												= tf.get_variable(dtype=__UpperCAmelCase							,  shape=tensor.shape							,  name=__UpperCAmelCase							,  initializer=tf.zeros_initializer()       )
  session.run(tf.variables_initializer([tf_var]       )       )
  session.run(__UpperCAmelCase       )
  return tf_var

 tf.reset_default_graph()
 with tf.Session() as session:
  for var_name in state_dict:
   a__												= to_tf_var_name(__UpperCAmelCase       )
   a__												= state_dict[var_name].numpy()
   if any(x in var_name for x in tensors_to_transpose       ):
    a__												= torch_tensor.T
   a__												= create_tf_var(tensor=__UpperCAmelCase							,  name=__UpperCAmelCase							,  session=__UpperCAmelCase       )
   tf.keras.backend.set_value(__UpperCAmelCase							,  __UpperCAmelCase       )
   a__												= session.run(__UpperCAmelCase       )
   print(f"Successfully created {tf_name}: {np.allclose(__UpperCAmelCase							,  __UpperCAmelCase       )}"       )

  a__												= tf.train.Saver(tf.trainable_variables()       )
  saver.save(__UpperCAmelCase							,  os.path.join(__UpperCAmelCase							,  model_name.replace('''-'''							,  '''_'''       ) + '''.ckpt'''       )       )






def     __a					(		__UpperCAmelCase=None       ):
 a__												= argparse.ArgumentParser()
 parser.add_argument('''--model_name'''							,  type=__UpperCAmelCase							,  required=__UpperCAmelCase							,  help='''model name e.g. bert-base-uncased'''       )
 parser.add_argument(
     '''--cache_dir'''							,  type=__UpperCAmelCase							,  default=__UpperCAmelCase							,  required=__UpperCAmelCase							,  help='''Directory containing pytorch model'''       )
 parser.add_argument('''--pytorch_model_path'''							,  type=__UpperCAmelCase							,  required=__UpperCAmelCase							,  help='''/path/to/<pytorch-model-name>.bin'''       )
 parser.add_argument('''--tf_cache_dir'''							,  type=__UpperCAmelCase							,  required=__UpperCAmelCase							,  help='''Directory in which to save tensorflow model'''       )
 a__												= parser.parse_args(__UpperCAmelCase       )

 a__												= BertModel.from_pretrained(
     pretrained_model_name_or_path=args.model_name							,  state_dict=torch.load(args.pytorch_model_path       )							,  cache_dir=args.cache_dir							,  )

 convert_pytorch_checkpoint_to_tf(model=__UpperCAmelCase							,  ckpt_dir=args.tf_cache_dir							,  model_name=args.model_name       )


if __name__ == "__main__":
     main()





