





'''simple docstring'''


from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand


def 						__a(SCREAMING_SNAKE_CASE_	:       Namespace     ):



    '''simple docstring'''





    return ConvertCommand(
        args.model_type		,							args.tf_checkpoint		,							args.pytorch_dump_output		,							args.config		,							args.finetuning_task_name     )


_SCREAMING_SNAKE_CASE									=				"\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"





class  lowerCAmelCase_		(		__magic_name__		):




 @staticmethod
 def 				_snake_case (						_lowerCAmelCase		)     ->				str:
     _lowerCAmelCase						      =   parser.add_parser(
         "convert"      ,  help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."      ,  )
     train_parser.add_argument("--model_type"      ,  type=_lowerCAmelCase      ,  required=_lowerCAmelCase      ,  help="Model's type."		)
     train_parser.add_argument(
         "--tf_checkpoint"      ,  type=_lowerCAmelCase      ,  required=_lowerCAmelCase      ,  help="TensorFlow checkpoint path or folder."		)
     train_parser.add_argument(
         "--pytorch_dump_output"      ,  type=_lowerCAmelCase      ,  required=_lowerCAmelCase      ,  help="Path to the PyTorch saved model output."		)
     train_parser.add_argument("--config"      ,  type=_lowerCAmelCase      ,  default=""      ,  help="Configuration file path or folder."		)
     train_parser.add_argument(
         "--finetuning_task_name"      ,  type=_lowerCAmelCase      ,  default=_lowerCAmelCase      ,  help="Optional fine-tuning task name if the TF model was a finetuned model."      ,  )
     train_parser.set_defaults(func=_lowerCAmelCase		)




 def __init__(						self      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  *_lowerCAmelCase      ,  )     ->				Dict:
     _lowerCAmelCase						      =   logging.get_logger("transformers-cli/converting"		)

     self._logger.info(f'''Loading model {model_type}'''		)
     _lowerCAmelCase						      =   model_type
     _lowerCAmelCase						      =   tf_checkpoint
     _lowerCAmelCase						      =   pytorch_dump_output
     _lowerCAmelCase						      =   config
     _lowerCAmelCase						      =   finetuning_task_name





 def 				_snake_case (						self		)     ->				str:
     if self._model_type == "albert":
         try:
             from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "bert":
         try:
             from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "funnel":
         try:
             from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "t5":
         try:
             from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "gpt":
         from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
             convert_openai_checkpoint_to_pytorch,
         )

         convert_openai_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "transfo_xl":
         try:
             from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                 convert_transfo_xl_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         if "ckpt" in self._tf_checkpoint.lower():
             _lowerCAmelCase						      =   self._tf_checkpoint
             _lowerCAmelCase						      =   ""
         else:
             _lowerCAmelCase						      =   self._tf_checkpoint
             _lowerCAmelCase						      =   ""
         convert_transfo_xl_checkpoint_to_pytorch(
             _lowerCAmelCase      ,  self._config      ,  self._pytorch_dump_output      ,  _lowerCAmelCase		)
     elif self._model_type == "gpt2":
         try:
             from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                 convert_gpta_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     elif self._model_type == "xlnet":
         try:
             from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                 convert_xlnet_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCAmelCase		)

         convert_xlnet_checkpoint_to_pytorch(
             self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output      ,  self._finetuning_task_name		)
     elif self._model_type == "xlm":
         from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
             convert_xlm_checkpoint_to_pytorch,
         )

         convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._pytorch_dump_output		)
     elif self._model_type == "lxmert":
         from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
             convert_lxmert_checkpoint_to_pytorch,
         )

         convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._pytorch_dump_output		)
     elif self._model_type == "rembert":
         from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
             convert_rembert_tf_checkpoint_to_pytorch,
         )

         convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint      ,  self._config      ,  self._pytorch_dump_output		)
     else:
         raise ValueError(
             "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"		)






import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
   from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
   from tensorflow.keras.optimizers import Adam


class      __lowerCAmelCase  (			tf.keras.optimizers.schedules.LearningRateSchedule       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							float     ,					_lowerCAmelCase							:							Callable     ,					_lowerCAmelCase							:							int     ,					_lowerCAmelCase							:							float = 1.0     ,					_lowerCAmelCase							:							str = None     ,					) ->			str:




     """simple docstring"""
     super().__init__()
     snake_case_            =       initial_learning_rate
     snake_case_            =       warmup_steps
     snake_case_            =       power
     snake_case_            =       decay_schedule_fn
     snake_case_            =       name



  def __call__(  self							:							Any     ,					_lowerCAmelCase							:							int	) ->			Tuple:




     """simple docstring"""
     with tf.name_scope(self.name or "WarmUp"	) as name:
        # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
        # learning rate will be `global_step/num_warmup_steps * init_lr`.
        snake_case_            =       tf.cast(_lowerCAmelCase     ,					tf.floataa	)
        snake_case_            =       tf.cast(self.warmup_steps     ,					tf.floataa	)
        snake_case_            =       global_step_float / warmup_steps_float
        snake_case_            =       self.initial_learning_rate * tf.math.pow(_lowerCAmelCase     ,					self.power	)
        return tf.cond(
            global_step_float < warmup_steps_float     ,					lambda: warmup_learning_rate     ,					lambda: self.decay_schedule_fn(step - self.warmup_steps	)     ,					name=_lowerCAmelCase     ,					)



  def        lowerCAmelCase__	(  self							:							Optional[Any]	) ->			Dict:




     """simple docstring"""
     return {
         "initial_learning_rate": self.initial_learning_rate,
         "decay_schedule_fn": self.decay_schedule_fn,
         "warmup_steps": self.warmup_steps,
         "power": self.power,
         "name": self.name,
     }






def       _lowerCAmelCase    (				lowerCAmelCase_				:float		,    lowerCAmelCase_				:int		,    lowerCAmelCase_				:int		,    lowerCAmelCase_				:float = 0.0		,    lowerCAmelCase_				:float = 0.9		,    lowerCAmelCase_				:float = 0.9_9_9		,    lowerCAmelCase_				:float = 1e-8		,    lowerCAmelCase_				:Optional[float] = None		,    lowerCAmelCase_				:Optional[float] = None		,    lowerCAmelCase_				:float = 0.0		,    lowerCAmelCase_				:float = 1.0		,    lowerCAmelCase_				:Optional[List[str]] = None		,    )->Optional[Any]:
   '''simple docstring'''






   snake_case_            =       tf.keras.optimizers.schedules.PolynomialDecay(
       initial_learning_rate=lowerCAmelCase_		,    decay_steps=num_train_steps - num_warmup_steps		,    end_learning_rate=init_lr * min_lr_ratio		,    power=lowerCAmelCase_		,    )
   if num_warmup_steps:
      snake_case_            =       WarmUp(
          initial_learning_rate=lowerCAmelCase_		,    decay_schedule_fn=lowerCAmelCase_		,    warmup_steps=lowerCAmelCase_		,    )
   if weight_decay_rate > 0.0:
      snake_case_            =       AdamWeightDecay(
          learning_rate=lowerCAmelCase_		,    weight_decay_rate=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    epsilon=lowerCAmelCase_		,    clipnorm=lowerCAmelCase_		,    global_clipnorm=lowerCAmelCase_		,    exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]		,    include_in_weight_decay=lowerCAmelCase_		,    )
   else:
      snake_case_            =       tf.keras.optimizers.Adam(
          learning_rate=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    epsilon=lowerCAmelCase_		,    clipnorm=lowerCAmelCase_		,    global_clipnorm=lowerCAmelCase_		,    )
   # We return the optimizer and the LR scheduler in order to better track the
   # evolution of the LR independently of the optimizer.
   return optimizer, lr_schedule


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Dict     ,					_lowerCAmelCase							:							Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001     ,					_lowerCAmelCase							:							float = 0.9     ,					_lowerCAmelCase							:							float = 0.999     ,					_lowerCAmelCase							:							float = 1e-7     ,					_lowerCAmelCase							:							bool = False     ,					_lowerCAmelCase							:							float = 0.0     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "AdamWeightDecay"     ,					**_lowerCAmelCase							:							Union[str, Any]     ,					) ->			Optional[int]:




     """simple docstring"""
     super().__init__(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       weight_decay_rate
     snake_case_            =       include_in_weight_decay
     snake_case_            =       exclude_from_weight_decay



  @classmethod
  def        lowerCAmelCase__	(  cls							:							Dict     ,					_lowerCAmelCase							:							Optional[Any]	) ->			Tuple:




     """simple docstring"""
     snake_case_            =       {"WarmUp": WarmUp}
     return super(_lowerCAmelCase     ,					cls	).from_config(_lowerCAmelCase     ,					custom_objects=_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							int     ,					_lowerCAmelCase							:							Optional[int]	) ->			List[str]:




     """simple docstring"""
     super(_lowerCAmelCase     ,					self	)._prepare_local(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     snake_case_            =       tf.constant(
         self.weight_decay_rate     ,					name="adam_weight_decay_rate"	)



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							List[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							List[Any]	) ->			int:




     """simple docstring"""
     snake_case_            =       self._do_use_weight_decay(var.name	)
     if do_decay:
        return var.assign_sub(
            learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["weight_decay_rate"]     ,					use_locking=self._use_locking     ,					)
     return tf.no_op()



  def        lowerCAmelCase__	(  self							:							List[str]     ,					_lowerCAmelCase							:							str     ,					_lowerCAmelCase							:							Optional[int]=None     ,					**_lowerCAmelCase							:							Dict	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       list(zip(*_lowerCAmelCase	)	)
     return super(_lowerCAmelCase     ,					self	).apply_gradients(zip(_lowerCAmelCase     ,					_lowerCAmelCase	)     ,					name=_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[str]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[int]	) ->			int:




     """simple docstring"""
     if apply_state is None:
        return self._decayed_lr_t[var_dtype], {}

     snake_case_            =       apply_state or {}
     snake_case_            =       apply_state.get((var_device, var_dtype)	)
     if coefficients is None:
        snake_case_            =       self._fallback_apply_state(_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       coefficients

     return coefficients["lr_t"], {"apply_state": apply_state}



  def        lowerCAmelCase__	(  self							:							int     ,					_lowerCAmelCase							:							List[Any]     ,					_lowerCAmelCase							:							Optional[int]     ,					_lowerCAmelCase							:							Tuple=None	) ->			List[str]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       self._get_lr(var.device     ,					var.dtype.base_dtype     ,					_lowerCAmelCase	)
     snake_case_            =       self._decay_weights_op(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     with tf.control_dependencies([decay]	):
        return super(_lowerCAmelCase     ,					self	)._resource_apply_dense(_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[Any]     ,					_lowerCAmelCase							:							Any     ,					_lowerCAmelCase							:							str     ,					_lowerCAmelCase							:							Any     ,					_lowerCAmelCase							:							Tuple=None	) ->			List[str]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       self._get_lr(var.device     ,					var.dtype.base_dtype     ,					_lowerCAmelCase	)
     snake_case_            =       self._decay_weights_op(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     with tf.control_dependencies([decay]	):
        return super(_lowerCAmelCase     ,					self	)._resource_apply_sparse(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							Union[str, Any]	) ->			int:




     """simple docstring"""
     snake_case_            =       super().get_config()
     config.update({"weight_decay_rate": self.weight_decay_rate}	)
     return config



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]	) ->			Tuple:




     """simple docstring"""
     if self.weight_decay_rate == 0:
        return False

     if self._include_in_weight_decay:
        for r in self._include_in_weight_decay:
           if re.search(_lowerCAmelCase     ,					_lowerCAmelCase	) is not None:
              return True

     if self._exclude_from_weight_decay:
        for r in self._exclude_from_weight_decay:
           if re.search(_lowerCAmelCase     ,					_lowerCAmelCase	) is not None:
              return False
     return True






class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							str	) ->			Dict:




     """simple docstring"""
     snake_case_            =       []
     snake_case_            =       None



  @property
  def        lowerCAmelCase__	(  self							:							Dict	) ->			List[Any]:




     """simple docstring"""
     if self._accum_steps is None:
        snake_case_            =       tf.Variable(
            tf.constant(0     ,					dtype=tf.intaa	)     ,					trainable=_lowerCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)

     return self._accum_steps.value()



  @property
  def        lowerCAmelCase__	(  self							:							Tuple	) ->			List[str]:




     """simple docstring"""
     if not self._gradients:
        raise ValueError("The accumulator should be called first to initialize the gradients"	)
     return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]



  def __call__(  self							:							List[Any]     ,					_lowerCAmelCase							:							Union[str, Any]	) ->			Optional[Any]:




     """simple docstring"""
     if not self._gradients:
        snake_case_            =       self.step  # Create the step variable.
        self._gradients.extend(
            [
                tf.Variable(
                    tf.zeros_like(_lowerCAmelCase	)     ,					trainable=_lowerCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)
                if gradient is not None
                else gradient
                for gradient in gradients
            ]	)
     if len(_lowerCAmelCase	) != len(self._gradients	):
        raise ValueError(F'''Expected {len(self._gradients	)} gradients, but got {len(_lowerCAmelCase	)}'''	)

     for accum_gradient, gradient in zip(self._gradients     ,					_lowerCAmelCase	):
        if accum_gradient is not None and gradient is not None:
           accum_gradient.assign_add(_lowerCAmelCase	)

     self._accum_steps.assign_add(1	)



  def        lowerCAmelCase__	(  self							:							Union[str, Any]	) ->			Optional[Any]:




     """simple docstring"""
     if not self._gradients:
        return
     self._accum_steps.assign(0	)
     for gradient in self._gradients:
        if gradient is not None:
           gradient.assign(tf.zeros_like(_lowerCAmelCase	)	)




