
"""simple docstring"""
from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand





def    _lowerCAmelCase    (						lowerCamelCase__							:							Namespace				)  ->    Optional[int]:
  return ConvertCommand(
      args.model_type,						args.tf_checkpoint,						args.pytorch_dump_output,						args.config,						args.finetuning_task_name				)


lowercase_       :						Optional[int]     					=	'''
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
'''





class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE  ):
     @staticmethod
     def 				__SCREAMING_SNAKE_CASE							(     snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = parser.add_parser(
           "convert"  ,     help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."  ,     )
       train_parser.add_argument("--model_type"  ,     type=snake_case__  ,     required=snake_case__  ,     help="Model\'s type."				)
       train_parser.add_argument(
           "--tf_checkpoint"  ,     type=snake_case__  ,     required=snake_case__  ,     help="TensorFlow checkpoint path or folder."				)
       train_parser.add_argument(
           "--pytorch_dump_output"  ,     type=snake_case__  ,     required=snake_case__  ,     help="Path to the PyTorch saved model output."				)
       train_parser.add_argument("--config"  ,     type=snake_case__  ,     default=""  ,     help="Configuration file path or folder."				)
       train_parser.add_argument(
           "--finetuning_task_name"  ,     type=snake_case__  ,     default=snake_case__  ,     help="Optional fine-tuning task name if the TF model was a finetuned model."  ,     )
       train_parser.set_defaults(func=snake_case__				)
     def __init__(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     *snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = logging.get_logger("transformers-cli/converting"				)

       self._logger.info(F'''Loading model {model_type}'''				)
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = model_type
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tf_checkpoint
       _SCREAMING_SNAKE_CASE   :				Dict							    = pytorch_dump_output
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = config
       _SCREAMING_SNAKE_CASE   :				Dict							    = finetuning_task_name




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       if self._model_type == "albert":
         try:
           from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
               convert_tf_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "bert":
         try:
           from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
               convert_tf_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "funnel":
         try:
           from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
               convert_tf_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "t5":
         try:
           from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
         except ImportError:
           raise ImportError(snake_case__				)

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "gpt":
         from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
             convert_openai_checkpoint_to_pytorch,
         )

         convert_openai_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "transfo_xl":
         try:
           from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
               convert_transfo_xl_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         if "ckpt" in self._tf_checkpoint.lower():
           _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self._tf_checkpoint
           _SCREAMING_SNAKE_CASE   :				Dict							    = ""
         else:
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = self._tf_checkpoint
           _SCREAMING_SNAKE_CASE   :				List[Any]							    = ""
         convert_transfo_xl_checkpoint_to_pytorch(
             snake_case__  ,     self._config  ,     self._pytorch_dump_output  ,     snake_case__				)
       elif self._model_type == "gpt2":
         try:
           from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
               convert_gpta_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       elif self._model_type == "xlnet":
         try:
           from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
               convert_xlnet_checkpoint_to_pytorch,
           )
         except ImportError:
           raise ImportError(snake_case__				)

         convert_xlnet_checkpoint_to_pytorch(
             self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output  ,     self._finetuning_task_name				)
       elif self._model_type == "xlm":
         from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
             convert_xlm_checkpoint_to_pytorch,
         )

         convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._pytorch_dump_output				)
       elif self._model_type == "lxmert":
         from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
             convert_lxmert_checkpoint_to_pytorch,
         )

         convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._pytorch_dump_output				)
       elif self._model_type == "rembert":
         from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
             convert_rembert_tf_checkpoint_to_pytorch,
         )

         convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint  ,     self._config  ,     self._pytorch_dump_output				)
       else:
         raise ValueError(
             "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"				)





"""simple docstring"""
import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE  ):
     A__      =			["""image_processor""", """tokenizer"""]
     A__      =			"""AutoImageProcessor"""
     A__      =			"""AutoTokenizer"""
     def __init__(     self  ,     snake_case__=None  ,     snake_case__=None  ,     **snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = None
       if "feature_extractor" in kwargs:
         warnings.warn(
             "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
             " instead."  ,     snake_case__  ,     )
         _SCREAMING_SNAKE_CASE   :				Any							    = kwargs.pop("feature_extractor"				)

       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
         raise ValueError("You need to specify an `image_processor`."				)
       if tokenizer is None:
         raise ValueError("You need to specify a `tokenizer`."				)

       super().__init__(snake_case__  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Any							    = self.image_processor
       _SCREAMING_SNAKE_CASE   :				Any							    = False
     def __call__(     self  ,     *snake_case__  ,     **snake_case__				):




       """simple docstring"""


       if self._in_target_context_manager:
         return self.current_processor(*snake_case__  ,     **snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = kwargs.pop("images"  ,     snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = kwargs.pop("text"  ,     snake_case__				)
       if len(snake_case__				) > 0:
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = args[0]
         _SCREAMING_SNAKE_CASE   :				List[str]							    = args[1:]

       if images is None and text is None:
         raise ValueError("You need to specify either an `images` or `text` input to process."				)

       if images is not None:
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = self.image_processor(snake_case__  ,     *snake_case__  ,     **snake_case__				)
       if text is not None:
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = self.tokenizer(snake_case__  ,     **snake_case__				)

       if text is None:
         return inputs
       elif images is None:
         return encodings
       else:
         _SCREAMING_SNAKE_CASE   :				List[Any]							    = encodings["input_ids"]
         return inputs
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     *snake_case__  ,     **snake_case__				):




       """simple docstring"""


       return self.tokenizer.batch_decode(*snake_case__  ,     **snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     *snake_case__  ,     **snake_case__				):




       """simple docstring"""


       return self.tokenizer.decode(*snake_case__  ,     **snake_case__				)
     @contextmanager
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       warnings.warn(
           "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
           "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
           "your images inputs, or in a separate call."				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = True
       _SCREAMING_SNAKE_CASE   :				str							    = self.tokenizer
       yield
       _SCREAMING_SNAKE_CASE   :				str							    = self.image_processor
       _SCREAMING_SNAKE_CASE   :				int							    = False
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__=False  ,     snake_case__=None				):




       """simple docstring"""


       if added_vocab is None:
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = self.tokenizer.get_added_vocab()

       _SCREAMING_SNAKE_CASE   :				int							    = {}

       while tokens:
         _SCREAMING_SNAKE_CASE   :				List[Any]							    = re.search(r"<s_(.*?)>"  ,     snake_case__  ,     re.IGNORECASE				)
         if start_token is None:
           break
         _SCREAMING_SNAKE_CASE   :				List[Any]							    = start_token.group(1				)
         _SCREAMING_SNAKE_CASE   :				Any							    = re.search(rF'''</s_{key}>'''  ,     snake_case__  ,     re.IGNORECASE				)
         _SCREAMING_SNAKE_CASE   :				Tuple							    = start_token.group()
         if end_token is None:
           _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tokens.replace(snake_case__  ,     ""				)
         else:
           _SCREAMING_SNAKE_CASE   :				Any							    = end_token.group()
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = re.escape(snake_case__				)
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = re.escape(snake_case__				)
           _SCREAMING_SNAKE_CASE   :				str							    = re.search(F'''{start_token_escaped}(.*?){end_token_escaped}'''  ,     snake_case__  ,     re.IGNORECASE				)
           if content is not None:
             _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = content.group(1				).strip()
             if r"<s_" in content and r"</s_" in content:  # non-leaf node
               _SCREAMING_SNAKE_CASE   :				Dict							    = self.tokenajson(snake_case__  ,     is_inner_value=snake_case__  ,     added_vocab=snake_case__				)
               if value:
                 if len(snake_case__				) == 1:
                   _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = value[0]
                 _SCREAMING_SNAKE_CASE   :				Dict							    = value
             else:  # leaf nodes
               _SCREAMING_SNAKE_CASE   :				Dict							    = []
               for leaf in content.split(r"<sep/>"				):
                 _SCREAMING_SNAKE_CASE   :				int							    = leaf.strip()
                 if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
                   _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = leaf[1:-2]  # for categorical special tokens
                 output[key].append(snake_case__				)
               if len(output[key]				) == 1:
                 _SCREAMING_SNAKE_CASE   :				Dict							    = output[key][0]

           _SCREAMING_SNAKE_CASE   :				List[Any]							    = tokens[tokens.find(snake_case__				) + len(snake_case__				) :].strip()
           if tokens[:6] == r"<sep/>":  # non-leaf nodes
             return [output] + self.tokenajson(tokens[6:]  ,     is_inner_value=snake_case__  ,     added_vocab=snake_case__				)

       if len(snake_case__				):
         return [output] if is_inner_value else output
       else:
         return [] if is_inner_value else {"text_sequence": tokens}
     @property
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       warnings.warn(
           "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."  ,     snake_case__  ,     )
       return self.image_processor_class




     @property
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       warnings.warn(
           "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."  ,     snake_case__  ,     )
       return self.image_processor



