
"""simple docstring"""







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowercase__		:    Tuple				   =     data_utils.TransfoXLTokenizer
lowercase__		:    Union[str, Any]				   =     data_utils.TransfoXLCorpus
lowercase__		:    int				   =     data_utils
lowercase__		:    Any				   =     data_utils

def     UpperCamelCase_   (   lowerCAmelCase__				:    Optional[int]   ,       lowerCAmelCase__				:    Union[str, Any]   ,       lowerCAmelCase__				:    List[Any]   ,       lowerCAmelCase__				:    Tuple						)						->       List[Any]:

  """simple docstring"""




  if transfo_xl_dataset_file:
    # Convert a pre-processed corpus (see original TensorFlow repo)
    with open(lowerCAmelCase__   ,       'rb'						) as fp:
      lowerCAmelCase_       :    Optional[Any]								=			pickle.load(lowerCAmelCase__   ,       encoding='latin1'						)
    # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
    lowerCAmelCase_       :    Union[str, Any]								=			pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
    print(f"Save vocabulary to {pytorch_vocab_dump_path}"						)
    lowerCAmelCase_       :    Optional[Any]								=			corpus.vocab.__dict__
    torch.save(lowerCAmelCase__   ,       lowerCAmelCase__						)

    lowerCAmelCase_       :    int								=			corpus.__dict__
    corpus_dict_no_vocab.pop('vocab'   ,       lowerCAmelCase__						)
    lowerCAmelCase_       :    str								=			pytorch_dump_folder_path + '/' + CORPUS_NAME
    print(f"Save dataset to {pytorch_dataset_dump_path}"						)
    torch.save(lowerCAmelCase__   ,       lowerCAmelCase__						)

  if tf_checkpoint_path:
    # Convert a pre-trained TensorFlow model
    lowerCAmelCase_       :    Any								=			os.path.abspath(lowerCAmelCase__						)
    lowerCAmelCase_       :    Tuple								=			os.path.abspath(lowerCAmelCase__						)

    print(f"Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."						)
    # Initialise PyTorch model
    if transfo_xl_config_file == "":
      lowerCAmelCase_       :    int								=			TransfoXLConfig()
    else:
      lowerCAmelCase_       :    Union[str, Any]								=			TransfoXLConfig.from_json_file(lowerCAmelCase__						)
    print(f"Building PyTorch model from configuration: {config}"						)
    lowerCAmelCase_       :    int								=			TransfoXLLMHeadModel(lowerCAmelCase__						)

    lowerCAmelCase_       :    Any								=			load_tf_weights_in_transfo_xl(lowerCAmelCase__   ,       lowerCAmelCase__   ,       lowerCAmelCase__						)
    # Save pytorch-model
    lowerCAmelCase_       :    List[Any]								=			os.path.join(lowerCAmelCase__   ,       lowerCAmelCase__						)
    lowerCAmelCase_       :    Union[str, Any]								=			os.path.join(lowerCAmelCase__   ,       lowerCAmelCase__						)
    print(f"Save PyTorch model to {os.path.abspath(lowerCAmelCase__						)}"						)
    torch.save(model.state_dict()   ,       lowerCAmelCase__						)
    print(f"Save configuration file to {os.path.abspath(lowerCAmelCase__						)}"						)
    with open(lowerCAmelCase__   ,       'w'   ,       encoding='utf-8'						) as f:
      f.write(config.to_json_string()						)


if __name__ == "__main__":
     lowercase__		:    int				   =     argparse.ArgumentParser()
     parser.add_argument(
         """--pytorch_dump_folder_path""",
         default=None,
         type=str,
         required=True,
         help="""Path to the folder to store the PyTorch model or dataset/vocab.""",
     )
     parser.add_argument(
         """--tf_checkpoint_path""",
         default="""""",
         type=str,
         help="""An optional path to a TensorFlow checkpoint path to be converted.""",
     )
     parser.add_argument(
         """--transfo_xl_config_file""",
         default="""""",
         type=str,
         help=(
             """An optional config json file corresponding to the pre-trained BERT model. \n"""
             """This specifies the model architecture."""
         ),
     )
     parser.add_argument(
         """--transfo_xl_dataset_file""",
         default="""""",
         type=str,
         help="""An optional dataset file to be converted in a vocabulary.""",
     )
     lowercase__		:    Optional[int]				   =     parser.parse_args()
     convert_transfo_xl_checkpoint_to_pytorch(
         args.tf_checkpoint_path,
         args.transfo_xl_config_file,
         args.pytorch_dump_folder_path,
         args.transfo_xl_dataset_file,
     )










"""simple docstring"""







import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


lowercase__		:    Tuple				   =     datasets.logging.get_logger(__name__)


lowercase__		:    List[Any]				   =     """\
@inproceedings{bleurt,
  title={BLEURT: Learning Robust Metrics for Text Generation},
  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},
  booktitle={ACL},
  year={2020},
  url={https://arxiv.org/abs/2004.04696}
}
"""

lowercase__		:    Tuple				   =     """\
BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)
and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune
it for your specific application (the latter is expected to perform better).

See the project's README at https://github.com/google-research/bleurt#readme for more information.
"""

lowercase__		:    List[Any]				   =     """
BLEURT score.

Args:
    `predictions` (list of str): prediction/candidate sentences
    `references` (list of str): reference sentences
    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.

Returns:
    'scores': List of scores.
Examples:

    >>> predictions = [\"hello there\", \"general kenobi\"]
    >>> references = [\"hello there\", \"general kenobi\"]
    >>> bleurt = datasets.load_metric(\"bleurt\")
    >>> results = bleurt.compute(predictions=predictions, references=references)
    >>> print([round(v, 2) for v in results[\"scores\"]])
    [1.03, 1.04]
"""

lowercase__		:    List[Any]				   =     {
    """bleurt-tiny-128""": """https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip""",
    """bleurt-tiny-512""": """https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip""",
    """bleurt-base-128""": """https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip""",
    """bleurt-base-512""": """https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip""",
    """bleurt-large-128""": """https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip""",
    """bleurt-large-512""": """https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip""",
    """BLEURT-20-D3""": """https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip""",
    """BLEURT-20-D6""": """https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip""",
    """BLEURT-20-D12""": """https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip""",
    """BLEURT-20""": """https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip""",
}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION  )
class 				UpperCamelCase__       (     datasets.Metric  ):




      """simple docstring"""




      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        return datasets.MetricInfo(
            description=_DESCRIPTION   ,      citation=_CITATION   ,      homepage='https://github.com/google-research/bleurt'   ,      inputs_description=_KWARGS_DESCRIPTION   ,      features=datasets.Features(
                {
                    'predictions': datasets.Value('string'   ,      id='sequence'	),
                    'references': datasets.Value('string'   ,      id='sequence'	),
                }	)   ,      codebase_urls=['https://github.com/google-research/bleurt']   ,      reference_urls=['https://github.com/google-research/bleurt', 'https://arxiv.org/abs/2004.04696']   ,      )



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				str	):
        # check that config name specifies a valid BLEURT model
        if self.config_name == "default":
          logger.warning(
              'Using default BLEURT-Base checkpoint for sequence maximum length 128. '
              'You can use a bigger model for better results with e.g.: datasets.load_metric(\'bleurt\', \'bleurt-large-512\').'	)
          lowerCAmelCase_       :    List[Any]								=			'bleurt-base-128'

        if self.config_name.lower() in CHECKPOINT_URLS:
          lowerCAmelCase_       :    List[Any]								=			self.config_name.lower()

        elif self.config_name.upper() in CHECKPOINT_URLS:
          lowerCAmelCase_       :    Tuple								=			self.config_name.upper()

        else:
          raise KeyError(
              F"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}"	)

        # download the model checkpoint specified by self.config_name and set up the scorer
        lowerCAmelCase_       :    List[Any]								=			dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]	)
        lowerCAmelCase_       :    List[str]								=			score.BleurtScorer(os.path.join(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	)	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Any	):
        lowerCAmelCase_       :    Tuple								=			self.scorer.score(references=SCREAMING_SNAKE_CASE_   ,      candidates=SCREAMING_SNAKE_CASE_	)
        return {"scores": scores}





