



"""simple docstring"""







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowercase__		:    Tuple				   =     data_utils.TransfoXLTokenizer
lowercase__		:    Union[str, Any]				   =     data_utils.TransfoXLCorpus
lowercase__		:    int				   =     data_utils
lowercase__		:    Any				   =     data_utils

def     UpperCamelCase_   (   lowerCAmelCase__				:    Optional[int]   ,       lowerCAmelCase__				:    Union[str, Any]   ,       lowerCAmelCase__				:    List[Any]   ,       lowerCAmelCase__				:    Tuple						)						->       List[Any]:

  """simple docstring"""




  if transfo_xl_dataset_file:
    # Convert a pre-processed corpus (see original TensorFlow repo)
    with open(lowerCAmelCase__   ,       'rb'						) as fp:
      lowerCAmelCase_       :    Optional[Any]								=			pickle.load(lowerCAmelCase__   ,       encoding='latin1'						)
    # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
    lowerCAmelCase_       :    Union[str, Any]								=			pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
    print(f"Save vocabulary to {pytorch_vocab_dump_path}"						)
    lowerCAmelCase_       :    Optional[Any]								=			corpus.vocab.__dict__
    torch.save(lowerCAmelCase__   ,       lowerCAmelCase__						)

    lowerCAmelCase_       :    int								=			corpus.__dict__
    corpus_dict_no_vocab.pop('vocab'   ,       lowerCAmelCase__						)
    lowerCAmelCase_       :    str								=			pytorch_dump_folder_path + '/' + CORPUS_NAME
    print(f"Save dataset to {pytorch_dataset_dump_path}"						)
    torch.save(lowerCAmelCase__   ,       lowerCAmelCase__						)

  if tf_checkpoint_path:
    # Convert a pre-trained TensorFlow model
    lowerCAmelCase_       :    Any								=			os.path.abspath(lowerCAmelCase__						)
    lowerCAmelCase_       :    Tuple								=			os.path.abspath(lowerCAmelCase__						)

    print(f"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."						)
    # Initialise PyTorch model
    if transfo_xl_config_file == "":
      lowerCAmelCase_       :    int								=			TransfoXLConfig()
    else:
      lowerCAmelCase_       :    Union[str, Any]								=			TransfoXLConfig.from_json_file(lowerCAmelCase__						)
    print(f"Building PyTorch model from configuration: {config}"						)
    lowerCAmelCase_       :    int								=			TransfoXLLMHeadModel(lowerCAmelCase__						)

    lowerCAmelCase_       :    Any								=			load_tf_weights_in_transfo_xl(lowerCAmelCase__   ,       lowerCAmelCase__   ,       lowerCAmelCase__						)
    # Save pytorch-model
    lowerCAmelCase_       :    List[Any]								=			os.path.join(lowerCAmelCase__   ,       lowerCAmelCase__						)
    lowerCAmelCase_       :    Union[str, Any]								=			os.path.join(lowerCAmelCase__   ,       lowerCAmelCase__						)
    print(f"Save PyTorch model to {os.path.abspath(lowerCAmelCase__						)}"						)
    torch.save(model.state_dict()   ,       lowerCAmelCase__						)
    print(f"Save configuration file to {os.path.abspath(lowerCAmelCase__						)}"						)
    with open(lowerCAmelCase__   ,       'w'   ,       encoding='utf-8'						) as f:
      f.write(config.to_json_string()						)


if __name__ == "__main__":
     lowercase__		:    int				   =     argparse.ArgumentParser()
     parser.add_argument(
         """--pytorch_dump_folder_path""",
         default=None,
         type=str,
         required=True,
         help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
     )
     parser.add_argument(
         """--tf_checkpoint_path""",
         default="""""",
         type=str,
         help="""An optional path to a TensorFlow checkpoint path to be converted.""",
     )
     parser.add_argument(
         """--transfo_xl_config_file""",
         default="""""",
         type=str,
         help=(
             """An optional config json file corresponding to the pre-trained BERT model. \n"""
             """This specifies the model architecture."""
         ),
     )
     parser.add_argument(
         """--transfo_xl_dataset_file""",
         default="""""",
         type=str,
         help="""An optional dataset file to be converted in a vocabulary.""",
     )
     lowercase__		:    Optional[int]				   =     parser.parse_args()
     convert_transfo_xl_checkpoint_to_pytorch(
         args.tf_checkpoint_path,
         args.transfo_xl_config_file,
         args.pytorch_dump_folder_path,
         args.transfo_xl_dataset_file,
     )










"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase__		:    Dict				   =     {
    """configuration_falcon""": ["""FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP""", """FalconConfig"""],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:    str				   =     [
         """FALCON_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """FalconForCausalLM""",
         """FalconModel""",
         """FalconPreTrainedModel""",
         """FalconForSequenceClassification""",
         """FalconForTokenClassification""",
         """FalconForQuestionAnswering""",
     ]


if TYPE_CHECKING:
     from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_falcon import (
              FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
              FalconForCausalLM,
              FalconForQuestionAnswering,
              FalconForSequenceClassification,
              FalconForTokenClassification,
              FalconModel,
              FalconPreTrainedModel,
          )


else:
     import sys

     lowercase__		:    Dict				   =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





