





"""simple docstring"""



from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand







def 						__lowercase	(  snake_case_    :    Union[str, Any]		)  ->Optional[int]:



    '''simple docstring'''







    return ConvertCommand(
        args.model_type							,args.tf_checkpoint							,args.pytorch_dump_output							,args.config							,args.finetuning_task_name		)


a_					=     """\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"""






class 						__snake_case   (						__lowerCAmelCase	):





 """simple docstring"""


 @staticmethod
 def    UpperCamelCase__(       __lowerCamelCase    ):



     '''simple docstring'''




     __A       :		List[str]							=	parser.add_parser(
         '''convert'''   ,		help='''CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'''   ,		)
     train_parser.add_argument('''--model_type'''   ,		type=_lowerCamelCase   ,		required=_lowerCamelCase   ,		help='''Model\'s type.'''    )
     train_parser.add_argument(
         '''--tf_checkpoint'''   ,		type=_lowerCamelCase   ,		required=_lowerCamelCase   ,		help='''TensorFlow checkpoint path or folder.'''    )
     train_parser.add_argument(
         '''--pytorch_dump_output'''   ,		type=_lowerCamelCase   ,		required=_lowerCamelCase   ,		help='''Path to the PyTorch saved model output.'''    )
     train_parser.add_argument('''--config'''   ,		type=_lowerCamelCase   ,		default=''''''   ,		help='''Configuration file path or folder.'''    )
     train_parser.add_argument(
         '''--finetuning_task_name'''   ,		type=_lowerCamelCase   ,		default=_lowerCamelCase   ,		help='''Optional fine-tuning task name if the TF model was a finetuned model.'''   ,		)
     train_parser.set_defaults(func=_lowerCamelCase    )

 def __init__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		*__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Dict							=	logging.get_logger('''transformers-cli/converting'''    )

     self._logger.info(F"""Loading model {model_type}"""    )
     __A       :		Optional[Any]							=	model_type
     __A       :		Union[str, Any]							=	tf_checkpoint
     __A       :		Dict							=	pytorch_dump_output
     __A       :		List[Any]							=	config
     __A       :		List[Any]							=	finetuning_task_name







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     if self._model_type == "albert":
         try:
             from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "bert":
         try:
             from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "funnel":
         try:
             from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                 convert_tf_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "t5":
         try:
             from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "gpt":
         from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
             convert_openai_checkpoint_to_pytorch,
         )

         convert_openai_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "transfo_xl":
         try:
             from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                 convert_transfo_xl_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         if "ckpt" in self._tf_checkpoint.lower():
             __A       :		Union[str, Any]							=	self._tf_checkpoint
             __A       :		Dict							=	''''''
         else:
             __A       :		Optional[int]							=	self._tf_checkpoint
             __A       :		Any							=	''''''
         convert_transfo_xl_checkpoint_to_pytorch(
             _lowerCamelCase   ,		self._config   ,		self._pytorch_dump_output   ,		_lowerCamelCase    )
     elif self._model_type == "gpt2":
         try:
             from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                 convert_gpta_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     elif self._model_type == "xlnet":
         try:
             from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                 convert_xlnet_checkpoint_to_pytorch,
             )
         except ImportError:
             raise ImportError(_lowerCamelCase    )

         convert_xlnet_checkpoint_to_pytorch(
             self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output   ,		self._finetuning_task_name    )
     elif self._model_type == "xlm":
         from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
             convert_xlm_checkpoint_to_pytorch,
         )

         convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._pytorch_dump_output    )
     elif self._model_type == "lxmert":
         from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
             convert_lxmert_checkpoint_to_pytorch,
         )

         convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._pytorch_dump_output    )
     elif self._model_type == "rembert":
         from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
             convert_rembert_tf_checkpoint_to_pytorch,
         )

         convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint   ,		self._config   ,		self._pytorch_dump_output    )
     else:
         raise ValueError(
             '''--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]'''    )





"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_albert import AlbertTokenizer
else:
	a				=None

a				=logging.get_logger(__name__)
a				={'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a				={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

a				={
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

a				='▁'






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  Dict    				=			VOCAB_FILES_NAMES
				A__ :  List[Any]    				=			PRETRAINED_VOCAB_FILES_MAP
				A__ :  List[Any]    				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__ :  Union[str, Any]    				=			AlbertTokenizer

				def __init__( self      ,					_lowerCamelCase=None      ,					_lowerCamelCase=None      ,					_lowerCamelCase=True      ,					_lowerCamelCase=True      ,					_lowerCamelCase=False      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<unk>"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<pad>"      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[MASK]"      ,					**_lowerCamelCase      ,					):
										# Mask token behave like a normal word, i.e. include the space before it and
										# is included in the raw text, there should be a match in a non-normalized sentence.
										lowerCamelCase__   =(
										    AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase      ,					normalized=_lowerCamelCase )
										    if isinstance(_lowerCamelCase      ,					_lowerCamelCase )
										    else mask_token
										)

										super().__init__(
										    _lowerCamelCase      ,					tokenizer_file=_lowerCamelCase      ,					do_lower_case=_lowerCamelCase      ,					remove_space=_lowerCamelCase      ,					keep_accents=_lowerCamelCase      ,					bos_token=_lowerCamelCase      ,					eos_token=_lowerCamelCase      ,					unk_token=_lowerCamelCase      ,					sep_token=_lowerCamelCase      ,					pad_token=_lowerCamelCase      ,					cls_token=_lowerCamelCase      ,					mask_token=_lowerCamelCase      ,					**_lowerCamelCase      ,					)

										lowerCamelCase__   =do_lower_case
										lowerCamelCase__   =remove_space
										lowerCamelCase__   =keep_accents
										lowerCamelCase__   =vocab_file
										lowerCamelCase__   =False if not self.vocab_file else True

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer." )

										if not os.path.isdir(_lowerCamelCase ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
																return
										lowerCamelCase__   =os.path.join(
										    _lowerCamelCase      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCamelCase ):
																copyfile(self.vocab_file      ,					_lowerCamelCase )

										return (out_vocab_file,)


