
from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand


def 							__A(lowerCAmelCase )		->				List[str]:

		"""simple docstring"""






		return ConvertCommand(
		    args.model_type						,  args.tf_checkpoint						,  args.pytorch_dump_output						,  args.config						,  args.finetuning_task_name )


lowerCamelCase__	=							"\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"







class        lowerCAmelCase__      (   __lowercase	):

						@staticmethod
						def  A_				(			a				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	parser.add_parser(
								    """convert"""       ,							help="""CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."""       ,							)
								train_parser.add_argument("""--model_type"""       ,							type=a       ,							required=a       ,							help="""Model's type."""				)
								train_parser.add_argument(
								    """--tf_checkpoint"""       ,							type=a       ,							required=a       ,							help="""TensorFlow checkpoint path or folder."""				)
								train_parser.add_argument(
								    """--pytorch_dump_output"""       ,							type=a       ,							required=a       ,							help="""Path to the PyTorch saved model output."""				)
								train_parser.add_argument("""--config"""       ,							type=a       ,							default=""""""       ,							help="""Configuration file path or folder."""				)
								train_parser.add_argument(
								    """--finetuning_task_name"""       ,							type=a       ,							default=a       ,							help="""Optional fine-tuning task name if the TF model was a finetuned model."""       ,							)
								train_parser.set_defaults(func=a				)

						def __init__(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							*a       ,							)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	logging.get_logger("""transformers-cli/converting"""				)

								self._logger.info(F'Loading model {model_type}'				)
								_UpperCamelCase						    =	model_type
								_UpperCamelCase						    =	tf_checkpoint
								_UpperCamelCase						    =	pytorch_dump_output
								_UpperCamelCase						    =	config
								_UpperCamelCase						    =	finetuning_task_name







						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								if self._model_type == "albert":
										try:
												from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
												    convert_tf_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "bert":
										try:
												from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
												    convert_tf_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "funnel":
										try:
												from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
												    convert_tf_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "t5":
										try:
												from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
										except ImportError:
												raise ImportError(a				)

										convert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "gpt":
										from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
										    convert_openai_checkpoint_to_pytorch,
										)

										convert_openai_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "transfo_xl":
										try:
												from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
												    convert_transfo_xl_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										if "ckpt" in self._tf_checkpoint.lower():
												_UpperCamelCase						    =	self._tf_checkpoint
												_UpperCamelCase						    =	""""""
										else:
												_UpperCamelCase						    =	self._tf_checkpoint
												_UpperCamelCase						    =	""""""
										convert_transfo_xl_checkpoint_to_pytorch(
										    a       ,							self._config       ,							self._pytorch_dump_output       ,							a				)
								elif self._model_type == "gpt2":
										try:
												from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
												    convert_gpta_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								elif self._model_type == "xlnet":
										try:
												from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
												    convert_xlnet_checkpoint_to_pytorch,
												)
										except ImportError:
												raise ImportError(a				)

										convert_xlnet_checkpoint_to_pytorch(
										    self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output       ,							self._finetuning_task_name				)
								elif self._model_type == "xlm":
										from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
										    convert_xlm_checkpoint_to_pytorch,
										)

										convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._pytorch_dump_output				)
								elif self._model_type == "lxmert":
										from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
										    convert_lxmert_checkpoint_to_pytorch,
										)

										convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._pytorch_dump_output				)
								elif self._model_type == "rembert":
										from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
										    convert_rembert_tf_checkpoint_to_pytorch,
										)

										convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint       ,							self._config       ,							self._pytorch_dump_output				)
								else:
										raise ValueError(
										    """--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]"""				)





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__	=							get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class        lowerCAmelCase__      (   __lowercase		,	unittest.TestCase	):
						UpperCamelCase_							: List[Any]	=		XLNetTokenizer
						UpperCamelCase_							: Optional[int]	=		XLNetTokenizerFast
						UpperCamelCase_							: Dict	=		True
						UpperCamelCase_							: Dict	=		True

						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								super().setUp()

								# We have a SentencePiece fixture for testing
								_UpperCamelCase						    =	XLNetTokenizer(a       ,							keep_accents=a				)
								tokenizer.sanitize_special_tokens()
								tokenizer.save_pretrained(self.tmpdirname				)

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	"""<s>"""
								_UpperCamelCase						    =	1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(a				)       ,							a				)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(a				)       ,							a				)

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	list(self.get_tokenizer().get_vocab().keys()				)

								self.assertEqual(vocab_keys[0]       ,							"""<unk>"""				)
								self.assertEqual(vocab_keys[1]       ,							"""<s>"""				)
								self.assertEqual(vocab_keys[-1]       ,							"""<eod>"""				)
								self.assertEqual(len(a				)       ,							10_06				)

						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								self.assertEqual(self.get_tokenizer().vocab_size       ,							10_00				)

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	XLNetTokenizer(a       ,							keep_accents=a				)

								_UpperCamelCase						    =	tokenizer.tokenize("""This is a test"""				)
								self.assertListEqual(a       ,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)       ,							[2_85, 46, 10, 1_70, 3_82]				)

								_UpperCamelCase						    =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]       ,							)
								_UpperCamelCase						    =	tokenizer.convert_tokens_to_ids(a				)
								self.assertListEqual(a       ,							[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]				)

								_UpperCamelCase						    =	tokenizer.convert_ids_to_tokens(a				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]       ,							)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	XLNetTokenizer(a       ,							do_lower_case=a				)
								_UpperCamelCase						    =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """""",
								        """i""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """se""",
								        """.""",
								    ]       ,							)
								self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				)       ,							["""▁he""", """ll""", """o"""]				)

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	XLNetTokenizer(a       ,							do_lower_case=a				)
								_UpperCamelCase						    =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """se""",
								        """.""",
								    ]       ,							)

						@slow
						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	XLNetTokenizer.from_pretrained("""xlnet-base-cased"""				)

								_UpperCamelCase						    =	tokenizer.encode("""sequence builders"""       ,							add_special_tokens=a				)
								_UpperCamelCase						    =	tokenizer.encode("""multi-sequence build"""       ,							add_special_tokens=a				)

								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a				)
								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a       ,							a				)

								assert encoded_sentence == text + [4, 3]
								assert encoded_pair == text + [4] + text_a + [4, 3]







						@slow
						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	{"""input_ids""": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=a       ,							model_name="""xlnet-base-cased"""       ,							revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""       ,							)



