





import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


__A     =    [
    {"dataset": "wikipedia", "config_name": "20220301.de"},
    {"dataset": "wikipedia", "config_name": "20220301.en"},
    {"dataset": "wikipedia", "config_name": "20220301.fr"},
    {"dataset": "wikipedia", "config_name": "20220301.frr"},
    {"dataset": "wikipedia", "config_name": "20220301.it"},
    {"dataset": "wikipedia", "config_name": "20220301.simple"},
    {"dataset": "snli", "config_name": "plain_text"},
    {"dataset": "eli5", "config_name": "LFQA_reddit"},
    {"dataset": "wiki40b", "config_name": "en"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.compressed"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.no_index"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.multiset.no_index"},
    {"dataset": "natural_questions", "config_name": "default"},
]





def 					lowerCAmelCase_			(	__a=True		)					->		Union[str, Any]:







  """simple docstring"""





  if with_config:
    return [
        {
            "testcase_name": d["dataset"] + "/" + d["config_name"],
            "dataset": d["dataset"],
            "config_name": d["config_name"],
        }
        for d in DATASETS_ON_HF_GCP
    ]
  else:
    return [
        {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
    ]



@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__SCREAMING_SNAKE_CASE   )   )
class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      None
 lowercase_		=      None

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       Optional[Any]	,			UpperCAmelCase_    :       Tuple)	->Tuple:



   '''simple docstring'''



   with TemporaryDirectory() as tmp_dir:
     lowerCamelCase__:				Optional[Any]        =dataset_module_factory(UpperCAmelCase_	,			cache_dir=UpperCAmelCase_)

     lowerCamelCase__:				Dict        =import_main_class(dataset_module.module_path	,			dataset=UpperCAmelCase_)

     lowerCamelCase__:				DatasetBuilder        =builder_cls(
         cache_dir=UpperCAmelCase_	,			config_name=UpperCAmelCase_	,			hash=dataset_module.hash	,			)

     lowerCamelCase__:				List[Any]        ="/".join(
         [
             HF_GCP_BASE_URL,
             builder_instance._relative_data_dir(with_hash=UpperCAmelCase_).replace(os.sep	,			"/"),
             config.DATASET_INFO_FILENAME,
         ])
     lowerCamelCase__:				Dict        =cached_path(UpperCAmelCase_	,			cache_dir=UpperCAmelCase_)
     self.assertTrue(os.path.exists(UpperCAmelCase_))





@pytest.mark.integration
def 					lowerCAmelCase_			(	__a		)					->		Dict:







  """simple docstring"""





  lowerCamelCase__:				str        =tmp_path_factory.mktemp("test_hf_gcp"		) / "test_wikipedia_simple"
  lowerCamelCase__:				Dict        =dataset_module_factory("wikipedia"     ,   cache_dir=__a		)
  lowerCamelCase__:				List[Any]        =import_main_class(dataset_module.module_path		)
  lowerCamelCase__:				DatasetBuilder        =builder_cls(
      cache_dir=__a     ,   config_name="20220301.frr"     ,   hash=dataset_module.hash     ,   )
  # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
  lowerCamelCase__:				int        =None
  builder_instance.download_and_prepare()
  lowerCamelCase__:				int        =builder_instance.as_dataset()
  assert ds






@pytest.mark.integration
def 					lowerCAmelCase_			(	__a		)					->		List[str]:







  """simple docstring"""





  lowerCamelCase__:				Union[str, Any]        =dataset_module_factory("wikipedia"     ,   cache_dir=__a		)
  lowerCamelCase__:				Union[str, Any]        =import_main_class(dataset_module.module_path     ,   dataset=__a		)
  lowerCamelCase__:				DatasetBuilder        =builder_cls(
      cache_dir=__a     ,   config_name="20220301.frr"     ,   hash=dataset_module.hash     ,   )
  lowerCamelCase__:				Optional[int]        =builder_instance.as_streaming_dataset()
  assert ds
  assert isinstance(__a     ,   __a		)
  assert "train" in ds
  assert isinstance(ds["train"]     ,   __a		)
  assert next(iter(ds["train"]		)		)








from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand





def 					lowerCAmelCase_			(	__a		)					->		int:







  """simple docstring"""





  return ConvertCommand(
      args.model_type     ,   args.tf_checkpoint     ,   args.pytorch_dump_output     ,   args.config     ,   args.finetuning_task_name		)


__A     =    "\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"






class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 @staticmethod
 def 							SCREAMING_SNAKE_CASE_ (UpperCAmelCase_    :       ArgumentParser)	->str:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =parser.add_parser(
       "convert"	,			help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints."	,			)
   train_parser.add_argument("--model_type"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="Model's type.")
   train_parser.add_argument(
       "--tf_checkpoint"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="TensorFlow checkpoint path or folder.")
   train_parser.add_argument(
       "--pytorch_dump_output"	,			type=UpperCAmelCase_	,			required=UpperCAmelCase_	,			help="Path to the PyTorch saved model output.")
   train_parser.add_argument("--config"	,			type=UpperCAmelCase_	,			default=""	,			help="Configuration file path or folder.")
   train_parser.add_argument(
       "--finetuning_task_name"	,			type=UpperCAmelCase_	,			default=UpperCAmelCase_	,			help="Optional fine-tuning task name if the TF model was a finetuned model."	,			)
   train_parser.set_defaults(func=UpperCAmelCase_)

 def __init__(self    :       List[Any]	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       str	,			*UpperCAmelCase_    :       Optional[int]	,			)	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =logging.get_logger("transformers-cli/converting")

   self._logger.info(F"""Loading model {model_type}""")
   lowerCamelCase__:				Any        =model_type
   lowerCamelCase__:				Optional[int]        =tf_checkpoint
   lowerCamelCase__:				Any        =pytorch_dump_output
   lowerCamelCase__:				Union[str, Any]        =config
   lowerCamelCase__:				str        =finetuning_task_name







 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any])	->Tuple:



   '''simple docstring'''



   if self._model_type == "albert":
     try:
       from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "bert":
     try:
       from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "funnel":
     try:
       from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
           convert_tf_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "t5":
     try:
       from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "gpt":
     from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
         convert_openai_checkpoint_to_pytorch,
     )

     convert_openai_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "transfo_xl":
     try:
       from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
           convert_transfo_xl_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     if "ckpt" in self._tf_checkpoint.lower():
       lowerCamelCase__:				Tuple        =self._tf_checkpoint
       lowerCamelCase__:				List[str]        =""
     else:
       lowerCamelCase__:				Any        =self._tf_checkpoint
       lowerCamelCase__:				Dict        =""
     convert_transfo_xl_checkpoint_to_pytorch(
         UpperCAmelCase_	,			self._config	,			self._pytorch_dump_output	,			UpperCAmelCase_)
   elif self._model_type == "gpt2":
     try:
       from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
           convert_gpta_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   elif self._model_type == "xlnet":
     try:
       from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
           convert_xlnet_checkpoint_to_pytorch,
       )
     except ImportError:
       raise ImportError(UpperCAmelCase_)

     convert_xlnet_checkpoint_to_pytorch(
         self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output	,			self._finetuning_task_name)
   elif self._model_type == "xlm":
     from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
         convert_xlm_checkpoint_to_pytorch,
     )

     convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint	,			self._pytorch_dump_output)
   elif self._model_type == "lxmert":
     from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
         convert_lxmert_checkpoint_to_pytorch,
     )

     convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint	,			self._pytorch_dump_output)
   elif self._model_type == "rembert":
     from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
         convert_rembert_tf_checkpoint_to_pytorch,
     )

     convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint	,			self._config	,			self._pytorch_dump_output)
   else:
     raise ValueError(
         "--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]")

