





'''simple docstring'''






import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
      import sqlitea

      import sqlalchemy







class 		UpperCAmelCase   (				lowercase_):


  """simple docstring"""







  def __init__(				self    :					int							,	UpperCamelCase__    :					Union[str, "sqlalchemy.sql.Selectable"]							,	UpperCamelCase__    :					Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]							,	UpperCamelCase__    :					Optional[Features] = None							,	UpperCamelCase__    :					str = None							,	UpperCamelCase__    :					bool = False							,	**UpperCamelCase__    :					Tuple							,	)     ->						Dict:
    super().__init__(features=UpperCamelCase__							,	cache_dir=UpperCamelCase__							,	keep_in_memory=UpperCamelCase__							,	**UpperCamelCase__  )
    _UpperCamelCase       			=Sql(
        cache_dir=UpperCamelCase__							,	features=UpperCamelCase__							,	sql=UpperCamelCase__							,	con=UpperCamelCase__							,	**UpperCamelCase__							,	)




  def   UpperCamelCase__      (				self    :					Dict  )     ->						str:
    _UpperCamelCase       			=None
    _UpperCamelCase       			=None
    _UpperCamelCase       			=None
    _UpperCamelCase       			=None

    self.builder.download_and_prepare(
        download_config=UpperCamelCase__							,	download_mode=UpperCamelCase__							,	verification_mode=UpperCamelCase__							,	base_path=UpperCamelCase__							,	)

    # Build dataset for splits
    _UpperCamelCase       			=self.builder.as_dataset(
        split='''train'''							,	verification_mode=UpperCamelCase__							,	in_memory=self.keep_in_memory  )
    return dataset







class 		UpperCAmelCase   :


  """simple docstring"""







  def __init__(				self    :					Union[str, Any]							,	UpperCamelCase__    :					Dataset							,	UpperCamelCase__    :					str							,	UpperCamelCase__    :					Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]							,	UpperCamelCase__    :					Optional[int] = None							,	UpperCamelCase__    :					Optional[int] = None							,	**UpperCamelCase__    :					List[Any]							,	)     ->						Optional[int]:
    if num_proc is not None and num_proc <= 0:
      raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''  )

    _UpperCamelCase       			=dataset
    _UpperCamelCase       			=name
    _UpperCamelCase       			=con
    _UpperCamelCase       			=batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
    _UpperCamelCase       			=num_proc
    _UpperCamelCase       			=to_sql_kwargs


  def   UpperCamelCase__      (				self    :					Dict  )     ->						int:
    _UpperCamelCase       			=self.to_sql_kwargs.pop('''sql'''							,	UpperCamelCase__  )
    _UpperCamelCase       			=self.to_sql_kwargs.pop('''con'''							,	UpperCamelCase__  )
    _UpperCamelCase       			=self.to_sql_kwargs.pop('''index'''							,	UpperCamelCase__  )

    _UpperCamelCase       			=self._write(index=UpperCamelCase__							,	**self.to_sql_kwargs  )
    return written


  def   UpperCamelCase__      (				self    :					Optional[int]							,	UpperCamelCase__    :					List[Any]  )     ->						int:
    _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase       			=args
    _UpperCamelCase       			={**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
    _UpperCamelCase       			=query_table(
        table=self.dataset.data							,	key=slice(UpperCamelCase__							,	offset + self.batch_size  )							,	indices=self.dataset._indices							,	)
    _UpperCamelCase       			=batch.to_pandas()
    _UpperCamelCase       			=df.to_sql(self.name							,	self.con							,	index=UpperCamelCase__							,	**UpperCamelCase__  )
    return num_rows or len(UpperCamelCase__  )




  def   UpperCamelCase__      (				self    :					Union[str, Any]							,	UpperCamelCase__    :					Optional[int]							,	**UpperCamelCase__    :					Union[str, Any]  )     ->						int:
    _UpperCamelCase       			=0

    if self.num_proc is None or self.num_proc == 1:
      for offset in logging.tqdm(
          range(0							,	len(self.dataset  )							,	self.batch_size  )							,	unit='''ba'''							,	disable=not logging.is_progress_bar_enabled()							,	desc='''Creating SQL from Arrow format'''							,	):
        written += self._batch_sql((offset, index, to_sql_kwargs)  )
    else:
      _UpperCamelCase			,     _UpperCamelCase       			=len(self.dataset  ), self.batch_size
      with multiprocessing.Pool(self.num_proc  ) as pool:
        for num_rows in logging.tqdm(
            pool.imap(
                self._batch_sql							,	[(offset, index, to_sql_kwargs) for offset in range(0							,	UpperCamelCase__							,	UpperCamelCase__  )]							,	)							,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size							,	unit='''ba'''							,	disable=not logging.is_progress_bar_enabled()							,	desc='''Creating SQL from Arrow format'''							,	):
          written += num_rows

    return written








'''simple docstring'''






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
__lowerCamelCase						: List[Any]		=   data_utils.TransfoXLTokenizer
__lowerCamelCase						: str		=   data_utils.TransfoXLCorpus
__lowerCamelCase						: Optional[int]		=   data_utils
__lowerCamelCase						: int		=   data_utils


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if transfo_xl_dataset_file:
    # Convert a pre-processed corpus (see original TensorFlow repo)
    with open(__SCREAMING_SNAKE_CASE  ,      '''rb'''   ) as fp:
      _UpperCamelCase       			=pickle.load(__SCREAMING_SNAKE_CASE  ,      encoding='''latin1'''   )
    # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
    _UpperCamelCase       			=pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
    print(f'''Save vocabulary to {pytorch_vocab_dump_path}'''   )
    _UpperCamelCase       			=corpus.vocab.__dict__
    torch.save(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )

    _UpperCamelCase       			=corpus.__dict__
    corpus_dict_no_vocab.pop('''vocab'''  ,      __SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=pytorch_dump_folder_path + '''/''' + CORPUS_NAME
    print(f'''Save dataset to {pytorch_dataset_dump_path}'''   )
    torch.save(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )

  if tf_checkpoint_path:
    # Convert a pre-trained TensorFlow model
    _UpperCamelCase       			=os.path.abspath(__SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=os.path.abspath(__SCREAMING_SNAKE_CASE   )

    print(f'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''   )
    # Initialise PyTorch model
    if transfo_xl_config_file == "":
      _UpperCamelCase       			=TransfoXLConfig()
    else:
      _UpperCamelCase       			=TransfoXLConfig.from_json_file(__SCREAMING_SNAKE_CASE   )
    print(f'''Building PyTorch model from configuration: {config}'''   )
    _UpperCamelCase       			=TransfoXLLMHeadModel(__SCREAMING_SNAKE_CASE   )

    _UpperCamelCase       			=load_tf_weights_in_transfo_xl(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    # Save pytorch-model
    _UpperCamelCase       			=os.path.join(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=os.path.join(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
    print(f'''Save PyTorch model to {os.path.abspath(__SCREAMING_SNAKE_CASE   )}'''   )
    torch.save(model.state_dict()  ,      __SCREAMING_SNAKE_CASE   )
    print(f'''Save configuration file to {os.path.abspath(__SCREAMING_SNAKE_CASE   )}'''   )
    with open(__SCREAMING_SNAKE_CASE  ,      '''w'''  ,      encoding='''utf-8'''   ) as f:
      f.write(config.to_json_string()   )


if __name__ == "__main__":
      __lowerCamelCase						: Any		=   argparse.ArgumentParser()
      parser.add_argument(
          '--pytorch_dump_folder_path',
          default=None,
          type=str,
          required=True,
          help='Path to the folder to store the PyTorch model or dataset/vocab.',
      )
      parser.add_argument(
          '--tf_checkpoint_path',
          default='',
          type=str,
          help='An optional path to a TensorFlow checkpoint path to be converted.',
      )
      parser.add_argument(
          '--transfo_xl_config_file',
          default='',
          type=str,
          help=(
              'An optional config json file corresponding to the pre-trained BERT model. \n'
              'This specifies the model architecture.'
          ),
      )
      parser.add_argument(
          '--transfo_xl_dataset_file',
          default='',
          type=str,
          help='An optional dataset file to be converted in a vocabulary.',
      )
      __lowerCamelCase						: Optional[int]		=   parser.parse_args()
      convert_transfo_xl_checkpoint_to_pytorch(
          args.tf_checkpoint_path,
          args.transfo_xl_config_file,
          args.pytorch_dump_folder_path,
          args.transfo_xl_dataset_file,
      )






