

import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
      import sqlitea

      import sqlalchemy




class   UpperCamelCase			(  UpperCAmelCase_ ):

    def __init__(      self			:   List[Any]				,     snake_case__			:   int				,     snake_case__			:   Optional[int]				,     snake_case__			:   Optional[Any] = None				,     snake_case__			:   Tuple = None				,     snake_case__			:   Union[str, Any] = False				,     **snake_case__			:   Union[str, Any]				,     ):

     """simple docstring"""






     super().__init__(features=SCREAMING_SNAKE_CASE_				,     cache_dir=SCREAMING_SNAKE_CASE_				,     keep_in_memory=SCREAMING_SNAKE_CASE_				,     **SCREAMING_SNAKE_CASE_     )
     SCREAMING_SNAKE_CASE		    =      Sql(
         cache_dir=SCREAMING_SNAKE_CASE_				,     features=SCREAMING_SNAKE_CASE_				,     sql=SCREAMING_SNAKE_CASE_				,     con=SCREAMING_SNAKE_CASE_				,     **SCREAMING_SNAKE_CASE_				,     )




    def UpperCamelCase      (      self			:   Optional[int]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      None
     SCREAMING_SNAKE_CASE		    =      None
     SCREAMING_SNAKE_CASE		    =      None
     SCREAMING_SNAKE_CASE		    =      None

     self.builder.download_and_prepare(
         download_config=SCREAMING_SNAKE_CASE_				,     download_mode=SCREAMING_SNAKE_CASE_				,     verification_mode=SCREAMING_SNAKE_CASE_				,     base_path=SCREAMING_SNAKE_CASE_				,     )

     # Build dataset for splits
     SCREAMING_SNAKE_CASE		    =      self.builder.as_dataset(
         split='train'				,     verification_mode=SCREAMING_SNAKE_CASE_				,     in_memory=self.keep_in_memory     )
     return dataset




class   UpperCamelCase			:

    def __init__(      self			:   Union[str, Any]				,     snake_case__			:   List[Any]				,     snake_case__			:   Optional[Any]				,     snake_case__			:   int				,     snake_case__			:   Dict = None				,     snake_case__			:   List[Any] = None				,     **snake_case__			:   List[Any]				,     ):

     """simple docstring"""






     if num_proc is not None and num_proc <= 0:
      raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""     )

     SCREAMING_SNAKE_CASE		    =      dataset
     SCREAMING_SNAKE_CASE		    =      name
     SCREAMING_SNAKE_CASE		    =      con
     SCREAMING_SNAKE_CASE		    =      batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
     SCREAMING_SNAKE_CASE		    =      num_proc
     SCREAMING_SNAKE_CASE		    =      to_sql_kwargs

    def UpperCamelCase      (      self			:   Optional[Any]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      self.to_sql_kwargs.pop('sql'				,     SCREAMING_SNAKE_CASE_     )
     SCREAMING_SNAKE_CASE		    =      self.to_sql_kwargs.pop('con'				,     SCREAMING_SNAKE_CASE_     )
     SCREAMING_SNAKE_CASE		    =      self.to_sql_kwargs.pop('index'				,     SCREAMING_SNAKE_CASE_     )

     SCREAMING_SNAKE_CASE		    =      self._write(index=SCREAMING_SNAKE_CASE_				,     **self.to_sql_kwargs     )
     return written

    def UpperCamelCase      (      self			:   Any				,     snake_case__			:   str     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      args
     SCREAMING_SNAKE_CASE		    =      {**to_sql_kwargs, """if_exists""": """append"""} if offset > 0 else to_sql_kwargs
     SCREAMING_SNAKE_CASE		    =      query_table(
         table=self.dataset.data				,     key=slice(SCREAMING_SNAKE_CASE_				,     offset + self.batch_size     )				,     indices=self.dataset._indices				,     )
     SCREAMING_SNAKE_CASE		    =      batch.to_pandas()
     SCREAMING_SNAKE_CASE		    =      df.to_sql(self.name				,     self.con				,     index=SCREAMING_SNAKE_CASE_				,     **SCREAMING_SNAKE_CASE_     )
     return num_rows or len(SCREAMING_SNAKE_CASE_     )




    def UpperCamelCase      (      self			:   Optional[int]				,     snake_case__			:   str				,     **snake_case__			:   Optional[Any]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      0

     if self.num_proc is None or self.num_proc == 1:
      for offset in logging.tqdm(
          range(0				,     len(self.dataset     )				,     self.batch_size     )				,     unit='ba'				,     disable=not logging.is_progress_bar_enabled()				,     desc='Creating SQL from Arrow format'				,     ):
       written += self._batch_sql((offset, index, to_sql_kwargs)     )
     else:
      SCREAMING_SNAKE_CASE		    =      len(self.dataset     ), self.batch_size
      with multiprocessing.Pool(self.num_proc     ) as pool:
       for num_rows in logging.tqdm(
           pool.imap(
               self._batch_sql				,     [(offset, index, to_sql_kwargs) for offset in range(0				,     SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_     )]				,     )				,     total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size				,     unit='ba'				,     disable=not logging.is_progress_bar_enabled()				,     desc='Creating SQL from Arrow format'				,     ):
        written += num_rows

     return written



from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding



class    _snake_case (						UpperCAmelCase_				):
 __lowerCAmelCase	:  Any	   =      ['image_processor', 'tokenizer']
 __lowerCAmelCase	:  Union[str, Any]	   =      'AutoImageProcessor'
 __lowerCAmelCase	:  int	   =      'AutoTokenizer'


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   super().__init__(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
   lowercase__	: Union[str, Any]     		=   self.image_processor


 def __call__(       self       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if text is None and images is None:
     raise ValueError("""You have to specify either text or images. Both cannot be none.""")

   if text is not None:
     lowercase__	: List[str]     		=   self.tokenizer(SCREAMING_SNAKE_CASE_       ,	return_tensors=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

   if images is not None:
     lowercase__	: Optional[int]     		=   self.image_processor(SCREAMING_SNAKE_CASE_       ,	return_tensors=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

   if text is not None and images is not None:
     lowercase__	: Union[str, Any]     		=   image_features.pixel_values
     return encoding
   elif text is not None:
     return encoding
   else:
     return BatchEncoding(data=dict(**SCREAMING_SNAKE_CASE_)       ,	tensor_type=SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	*SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	*SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return self.tokenizer.decode(*SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)



 @property
 def   lowercase__  (       self):


   '''simple docstring'''




   return ["input_ids", "attention_mask", "pixel_values"]






