


import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__UpperCamelCase       :       List[Any]						       =				data_utils.TransfoXLTokenizer
__UpperCamelCase       :       Dict						       =				data_utils.TransfoXLCorpus
__UpperCamelCase       :       int						       =				data_utils
__UpperCamelCase       :       str						       =				data_utils

def 			_a  (				SCREAMING_SNAKE_CASE	:				List[str]    , SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				Any    , SCREAMING_SNAKE_CASE	:				List[Any]     ):







 """simple docstring"""



 if transfo_xl_dataset_file:
  # Convert a pre-processed corpus (see original TensorFlow repo)
  with open(SCREAMING_SNAKE_CASE    , '''rb'''     ) as fp:
   UpperCamelCase__					:    str          =   pickle.load(SCREAMING_SNAKE_CASE    , encoding='''latin1'''     )
  # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
  UpperCamelCase__					:    Optional[Any]          =   pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
  print(F"Save vocabulary to {pytorch_vocab_dump_path}"     )
  UpperCamelCase__					:    List[Any]          =   corpus.vocab.__dict__
  torch.save(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

  UpperCamelCase__					:    Any          =   corpus.__dict__
  corpus_dict_no_vocab.pop('''vocab'''    , SCREAMING_SNAKE_CASE     )
  UpperCamelCase__					:    Any          =   pytorch_dump_folder_path + '''/''' + CORPUS_NAME
  print(F"Save dataset to {pytorch_dataset_dump_path}"     )
  torch.save(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 if tf_checkpoint_path:
  # Convert a pre-trained TensorFlow model
  UpperCamelCase__					:    List[str]          =   os.path.abspath(SCREAMING_SNAKE_CASE     )
  UpperCamelCase__					:    Optional[int]          =   os.path.abspath(SCREAMING_SNAKE_CASE     )

  print(F"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."     )
  # Initialise PyTorch model
  if transfo_xl_config_file == "":
   UpperCamelCase__					:    Dict          =   TransfoXLConfig()
  else:
   UpperCamelCase__					:    Optional[int]          =   TransfoXLConfig.from_json_file(SCREAMING_SNAKE_CASE     )
  print(F"Building PyTorch model from configuration: {config}"     )
  UpperCamelCase__					:    str          =   TransfoXLLMHeadModel(SCREAMING_SNAKE_CASE     )

  UpperCamelCase__					:    Tuple          =   load_tf_weights_in_transfo_xl(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
  # Save pytorch-model
  UpperCamelCase__					:    Optional[Any]          =   os.path.join(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
  UpperCamelCase__					:    str          =   os.path.join(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
  print(F"Save PyTorch model to {os.path.abspath(SCREAMING_SNAKE_CASE     )}"     )
  torch.save(model.state_dict()    , SCREAMING_SNAKE_CASE     )
  print(F"Save configuration file to {os.path.abspath(SCREAMING_SNAKE_CASE     )}"     )
  with open(SCREAMING_SNAKE_CASE    , '''w'''    , encoding='''utf-8'''     ) as f:
   f.write(config.to_json_string()     )


if __name__ == "__main__":
       __UpperCamelCase       :       Any						       =				argparse.ArgumentParser()
       parser.add_argument(
           "--pytorch_dump_folder_path",
           default=None,
           type=str,
           required=True,
           help="Path to the folder to store the PyTorch model or dataset/vocab.",
       )
       parser.add_argument(
           "--tf_checkpoint_path",
           default="",
           type=str,
           help="An optional path to a TensorFlow checkpoint path to be converted.",
       )
       parser.add_argument(
           "--transfo_xl_config_file",
           default="",
           type=str,
           help=(
               "An optional config json file corresponding to the pre-trained BERT model. \n"
               "This specifies the model architecture."
           ),
       )
       parser.add_argument(
           "--transfo_xl_dataset_file",
           default="",
           type=str,
           help="An optional dataset file to be converted in a vocabulary.",
       )
       __UpperCamelCase       :       int						       =				parser.parse_args()
       convert_transfo_xl_checkpoint_to_pytorch(
           args.tf_checkpoint_path,
           args.transfo_xl_config_file,
           args.pytorch_dump_folder_path,
           args.transfo_xl_dataset_file,
       )






import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
       from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
       from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
       from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__UpperCamelCase       :       Dict						       =				DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__UpperCamelCase       :       Optional[int]						       =				"main"
# Default branch name
__UpperCamelCase       :       Optional[Any]						       =				"f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
__UpperCamelCase       :       str						       =				"aaaaaaa"
# This commit does not exist, so we should 404.

__UpperCamelCase       :       Optional[Any]						       =				"d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
__UpperCamelCase       :       Tuple						       =				"4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"

@contextlib.contextmanager
def 			_a  (				):







 """simple docstring"""



 print('''Welcome!'''     )
 yield
 print('''Bye!'''     )

@contextlib.contextmanager
def 			_a  (				):







 """simple docstring"""



 print('''Bonjour!'''     )
 yield
 print('''Au revoir!'''     )

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Any:





      '''simple docstring'''






      assert transformers.__spec__ is not None
      assert importlib.util.find_spec('''transformers'''			) is not None



class       __magic_name__	(     unittest.TestCase):


     @unittest.mock.patch('''sys.stdout'''							,	new_callable=io.StringIO			)
     def      UpperCAmelCase__	(      self							:     Union[str, Any]							,	lowerCamelCase__							:     Union[str, Any]			)  ->      Optional[Any]:





      '''simple docstring'''






      with ContextManagers([]			):
       print('''Transformers are awesome!'''			)
      # The print statement adds a new line at the end of the output
      self.assertEqual(mock_stdout.getvalue()							,	'''Transformers are awesome!\n'''			)


     @unittest.mock.patch('''sys.stdout'''							,	new_callable=io.StringIO			)
     def      UpperCAmelCase__	(      self							:     Any							,	lowerCamelCase__							:     Union[str, Any]			)  ->      Dict:





      '''simple docstring'''






      with ContextManagers([context_en()]			):
       print('''Transformers are awesome!'''			)
      # The output should be wrapped with an English welcome and goodbye
      self.assertEqual(mock_stdout.getvalue()							,	'''Welcome!\nTransformers are awesome!\nBye!\n'''			)


     @unittest.mock.patch('''sys.stdout'''							,	new_callable=io.StringIO			)
     def      UpperCAmelCase__	(      self							:     List[Any]							,	lowerCamelCase__							:     List[str]			)  ->      Tuple:





      '''simple docstring'''






      with ContextManagers([context_fr(), context_en()]			):
       print('''Transformers are awesome!'''			)
      # The output should be wrapped with an English and French welcome and goodbye
      self.assertEqual(mock_stdout.getvalue()							,	'''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''			)


     @require_torch
     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      Optional[int]:





      '''simple docstring'''






      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''']			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''', '''next_sentence_label''']			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''start_positions''', '''end_positions''']			)

      class       __magic_name__	(     __lowerCAmelCase):
           pass



      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''']			)


     @require_tf
     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Tuple:





      '''simple docstring'''






      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''']			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''', '''next_sentence_label''']			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''start_positions''', '''end_positions''']			)

      class       __magic_name__	(     __lowerCAmelCase):
           pass



      self.assertEqual(find_labels(lowerCamelCase__			)							,	['''labels''']			)


     @require_flax
     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[Any]:





      '''simple docstring'''






      self.assertEqual(find_labels(lowerCamelCase__			)							,	[]			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	[]			)
      self.assertEqual(find_labels(lowerCamelCase__			)							,	[]			)

      class       __magic_name__	(     __lowerCAmelCase):
           pass



      self.assertEqual(find_labels(lowerCamelCase__			)							,	[]			)


