






"""simple docstring"""






import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 							lowerCAmelCase__       (    UpperCAmelCase__       ,	unittest.TestCase				):






      '''simple docstring'''

      __UpperCamelCase							=						MgpstrTokenizer
      __UpperCamelCase							=						False
      __UpperCamelCase							=						{}
      __UpperCamelCase							=						False
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          super().setUp()

          # fmt: off
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
          # fmt: on
          SCREAMING_SNAKE_CASE_   :						Tuple							=	dict(zip(lowercase_		,			range(len(lowercase_))))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	os.path.join(self.tmpdirname		,			VOCAB_FILES_NAMES['''vocab_file'''])
          with open(self.vocab_file		,			'''w'''		,			encoding='''utf-8''') as fp:
              fp.write(json.dumps(lowercase_) + '''\n''')
      def _SCREAMING_SNAKE_CASE				(		self  :						str		,			**lowercase_  :						Tuple):




          '''simple docstring'''


          return MgpstrTokenizer.from_pretrained(self.tmpdirname		,			**lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Any		,			lowercase_  :						Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	'''tester'''
          SCREAMING_SNAKE_CASE_   :						Any							=	'''tester'''
          return input_text, output_text
      @unittest.skip('''MGP-STR always lower cases letters.''')
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          pass
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[Any]							=	self.get_tokenizers(do_lower_case=lowercase_)
          for tokenizer in tokenizers:
              with self.subTest(F'{tokenizer.__class__.__name__}'):
                  SCREAMING_SNAKE_CASE_   :						Optional[int]							=	'''[SPECIAL_TOKEN]'''

                  tokenizer.add_special_tokens({'''cls_token''': special_token})
                  SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tokenizer.encode([special_token]		,			add_special_tokens=lowercase_)
                  self.assertEqual(len(lowercase_)		,			1)

                  SCREAMING_SNAKE_CASE_   :						str							=	tokenizer.decode(lowercase_		,			skip_special_tokens=lowercase_)
                  self.assertTrue(special_token not in decoded)
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(F'{tokenizer.__class__.__name__}'):
                  SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_   :						Dict							=	self.get_input_output_texts(lowercase_)

                  SCREAMING_SNAKE_CASE_   :						int							=	tokenizer.tokenize(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer.convert_tokens_to_ids(lowercase_)
                  SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tokenizer.encode(lowercase_		,			add_special_tokens=lowercase_)
                  self.assertListEqual(lowercase_		,			lowercase_)

                  SCREAMING_SNAKE_CASE_   :						Any							=	tokenizer.convert_ids_to_tokens(lowercase_)
                  self.assertNotEqual(len(lowercase_)		,			0)
                  SCREAMING_SNAKE_CASE_   :						Dict							=	tokenizer.decode(lowercase_)
                  self.assertIsInstance(lowercase_		,			lowercase_)

                  self.assertEqual(text_a.replace(''' '''		,			'''''')		,			lowercase_)
      @unittest.skip('''MGP-STR tokenizer only handles one sequence.''')
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]):




          '''simple docstring'''


          pass



      @unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer''')
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]):




          '''simple docstring'''


          pass










"""simple docstring"""






import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


UpperCAmelCase_     :				int         =				{
    """cola""": 2,
    """mnli""": 3,
    """mrpc""": 2,
    """sst-2""": 2,
    """sts-b""": 1,
    """qqp""": 2,
    """qnli""": 2,
    """rte""": 2,
    """wnli""": 2,
}


logging.set_verbosity_info()



def       _A    (__a					,       __a					,       __a					,       __a=None							)				-> int:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						str							=	XLNetConfig.from_json_file(__a							)

    SCREAMING_SNAKE_CASE_   :						List[Any]							=	finetuning_task.lower() if finetuning_task is not None else ''''''
    if finetuning_task in GLUE_TASKS_NUM_LABELS:
        print(f'Building PyTorch XLNetForSequenceClassification model from configuration: {config}'							)
        SCREAMING_SNAKE_CASE_   :						List[Any]							=	finetuning_task
        SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	GLUE_TASKS_NUM_LABELS[finetuning_task]
        SCREAMING_SNAKE_CASE_   :						Dict							=	XLNetForSequenceClassification(__a							)
    elif "squad" in finetuning_task:
        SCREAMING_SNAKE_CASE_   :						str							=	finetuning_task
        SCREAMING_SNAKE_CASE_   :						Dict							=	XLNetForQuestionAnswering(__a							)
    else:
        SCREAMING_SNAKE_CASE_   :						List[str]							=	XLNetLMHeadModel(__a							)

    # Load weights from tf checkpoint
    load_tf_weights_in_xlnet(__a					,       __a					,       __a							)

    # Save pytorch-model
    SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	os.path.join(__a					,       __a							)
    SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	os.path.join(__a					,       __a							)
    print(f'Save PyTorch model to {os.path.abspath(__a							)}'							)
    torch.save(model.state_dict()					,       __a							)
    print(f'Save configuration file to {os.path.abspath(__a							)}'							)
    with open(__a					,       '''w'''					,       encoding='''utf-8'''							) as f:
        f.write(config.to_json_string()							)


if __name__ == "__main__":
  UpperCAmelCase_     :				str         =				argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
  )
  parser.add_argument(
      """--xlnet_config_file""",
      default=None,
      type=str,
      required=True,
      help=(
          """The config json file corresponding to the pre-trained XLNet model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default=None,
      type=str,
      required=True,
      help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
  )
  parser.add_argument(
      """--finetuning_task""",
      default=None,
      type=str,
      help="""Name of a task on which the XLNet TensorFlow model was fine-tuned""",
  )
  UpperCAmelCase_     :				int         =				parser.parse_args()
  print(args)

  convert_xlnet_checkpoint_to_pytorch(
      args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
  )


