






"""simple docstring"""






import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
  import sqlitea

  import sqlalchemy



class 							lowerCAmelCase__       (    UpperCAmelCase__				):






      '''simple docstring'''

      def __init__(		self  :						str		,			lowercase_  :						Union[str, "sqlalchemy.sql.Selectable"]		,			lowercase_  :						Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		,			lowercase_  :						Optional[Features] = None		,			lowercase_  :						str = None		,			lowercase_  :						bool = False		,			**lowercase_  :						Tuple		,			):




          '''simple docstring'''


          super().__init__(features=lowercase_		,			cache_dir=lowercase_		,			keep_in_memory=lowercase_		,			**lowercase_)
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	Sql(
              cache_dir=lowercase_		,			features=lowercase_		,			sql=lowercase_		,			con=lowercase_		,			**lowercase_		,			)



      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	None
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	None
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	None
          SCREAMING_SNAKE_CASE_   :						str							=	None

          self.builder.download_and_prepare(
              download_config=lowercase_		,			download_mode=lowercase_		,			verification_mode=lowercase_		,			base_path=lowercase_		,			)

          # Build dataset for splits
          SCREAMING_SNAKE_CASE_   :						List[str]							=	self.builder.as_dataset(
              split='''train'''		,			verification_mode=lowercase_		,			in_memory=self.keep_in_memory)
          return dataset



class 							lowerCAmelCase__       :






      '''simple docstring'''

      def __init__(		self  :						List[Any]		,			lowercase_  :						Dataset		,			lowercase_  :						str		,			lowercase_  :						Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		,			lowercase_  :						Optional[int] = None		,			lowercase_  :						Optional[int] = None		,			**lowercase_  :						Tuple		,			):




          '''simple docstring'''


          if num_proc is not None and num_proc <= 0:
              raise ValueError(F'num_proc {num_proc} must be an integer > 0.')

          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	dataset
          SCREAMING_SNAKE_CASE_   :						List[str]							=	name
          SCREAMING_SNAKE_CASE_   :						str							=	con
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	num_proc
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	to_sql_kwargs
      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	self.to_sql_kwargs.pop('''sql'''		,			lowercase_)
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	self.to_sql_kwargs.pop('''con'''		,			lowercase_)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	self.to_sql_kwargs.pop('''index'''		,			lowercase_)

          SCREAMING_SNAKE_CASE_   :						List[str]							=	self._write(index=lowercase_		,			**self.to_sql_kwargs)
          return written
      def _SCREAMING_SNAKE_CASE				(		self  :						int		,			lowercase_  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	args
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	{**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
          SCREAMING_SNAKE_CASE_   :						str							=	query_table(
              table=self.dataset.data		,			key=slice(lowercase_		,			offset + self.batch_size)		,			indices=self.dataset._indices		,			)
          SCREAMING_SNAKE_CASE_   :						int							=	batch.to_pandas()
          SCREAMING_SNAKE_CASE_   :						str							=	df.to_sql(self.name		,			self.con		,			index=lowercase_		,			**lowercase_)
          return num_rows or len(lowercase_)



      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]		,			lowercase_  :						Tuple		,			**lowercase_  :						Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Any							=	0

          if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0		,			len(self.dataset)		,			self.batch_size)		,			unit='''ba'''		,			disable=not logging.is_progress_bar_enabled()		,			desc='''Creating SQL from Arrow format'''		,			):
                  written += self._batch_sql((offset, index, to_sql_kwargs))
          else:
              SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	len(self.dataset), self.batch_size
              with multiprocessing.Pool(self.num_proc) as pool:
                  for num_rows in logging.tqdm(
                      pool.imap(
                          self._batch_sql		,			[(offset, index, to_sql_kwargs) for offset in range(0		,			lowercase_		,			lowercase_)]		,			)		,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,			unit='''ba'''		,			disable=not logging.is_progress_bar_enabled()		,			desc='''Creating SQL from Arrow format'''		,			):
                      written += num_rows

          return written










"""simple docstring"""






import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
  import jax.numpy as jnp

if is_tf_available():
  import tensorflow as tf

if is_torch_available():
  import torch




class 							lowerCAmelCase__       (    unittest.TestCase				):






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	{
              '''task_specific_params''': {
                  '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 128, '''min_length''': 12, '''num_beams''': 4},
                  '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 142, '''min_length''': 56, '''num_beams''': 4},
                  '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 62, '''min_length''': 11, '''num_beams''': 6},
              }
          }
          SCREAMING_SNAKE_CASE_   :						List[str]							=	{
              '''task_specific_params.summarization.length_penalty''': 1.0,
              '''task_specific_params.summarization.max_length''': 128,
              '''task_specific_params.summarization.min_length''': 12,
              '''task_specific_params.summarization.num_beams''': 4,
              '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
              '''task_specific_params.summarization_cnn.max_length''': 142,
              '''task_specific_params.summarization_cnn.min_length''': 56,
              '''task_specific_params.summarization_cnn.num_beams''': 4,
              '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
              '''task_specific_params.summarization_xsum.max_length''': 62,
              '''task_specific_params.summarization_xsum.min_length''': 11,
              '''task_specific_params.summarization_xsum.num_beams''': 6,
          }

          self.assertEqual(flatten_dict(lowercase_)		,			lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			x.transpose()))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4		,			5)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			x.transpose((1, 2, 0))))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						str							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			transpose(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Dict							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			transpose(lowercase_		,			axes=(1, 2, 0)).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						str							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			transpose(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						int							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			transpose(lowercase_		,			axes=(1, 2, 0)).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						Any							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			np.asarray(transpose(lowercase_))))

          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Any							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			np.asarray(transpose(lowercase_		,			axes=(1, 2, 0)))))
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			np.reshape(lowercase_		,			(4, 3))))

          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4		,			5)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			np.reshape(lowercase_		,			(12, 5))))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						List[str]							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			reshape(lowercase_		,			(4, 3)).numpy()))

          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			reshape(lowercase_		,			(12, 5)).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			reshape(lowercase_		,			(4, 3)).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			reshape(lowercase_		,			(12, 5)).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Any							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			np.asarray(reshape(lowercase_		,			(4, 3)))))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			np.asarray(reshape(lowercase_		,			(12, 5)))))
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(1		,			3		,			4)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			np.squeeze(lowercase_)))

          SCREAMING_SNAKE_CASE_   :						str							=	np.random.randn(1		,			4		,			1		,			5)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			np.squeeze(lowercase_		,			axis=2)))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			squeeze(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						str							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			squeeze(lowercase_		,			axis=2).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						Dict							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			squeeze(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						Any							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			squeeze(lowercase_		,			axis=2).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			np.asarray(squeeze(lowercase_))))

          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						str							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			np.asarray(squeeze(lowercase_		,			axis=2))))
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			np.expand_dims(lowercase_		,			axis=1)))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			expand_dims(lowercase_		,			axis=1).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						Any							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			expand_dims(lowercase_		,			axis=1).numpy()))



      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			np.asarray(expand_dims(lowercase_		,			axis=1))))


