


'''simple docstring'''
import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
  import sqlitea

  import sqlalchemy




class      lowerCamelCase_  (      __a ):






      def __init__(   self							:       Optional[int]		,		_A							:       Union[str, "sqlalchemy.sql.Selectable"]		,		_A							:       Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		,		_A							:       Optional[Features] = None		,		_A							:       str = None		,		_A							:       bool = False		,		**_A							:       Dict		,		):


          '''simple docstring'''


          super().__init__(features=_A		,		cache_dir=_A		,		keep_in_memory=_A		,		**_A					)
          UpperCAmelCase__     :						int						=							Sql(
              cache_dir=_A		,		features=_A		,		sql=_A		,		con=_A		,		**_A		,		)






      def lowercase_				(   self							:       List[str]					):


          '''simple docstring'''


          UpperCAmelCase__     :						Union[str, Any]						=							None
          UpperCAmelCase__     :						Tuple						=							None
          UpperCAmelCase__     :						List[str]						=							None
          UpperCAmelCase__     :						List[str]						=							None

          self.builder.download_and_prepare(
              download_config=_A		,		download_mode=_A		,		verification_mode=_A		,		base_path=_A		,		)

          # Build dataset for splits
          UpperCAmelCase__     :						Optional[int]						=							self.builder.as_dataset(
              split='''train'''		,		verification_mode=_A		,		in_memory=self.keep_in_memory					)
          return dataset




class      lowerCamelCase_  :






      def __init__(   self							:       Any		,		_A							:       Dataset		,		_A							:       str		,		_A							:       Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		,		_A							:       Optional[int] = None		,		_A							:       Optional[int] = None		,		**_A							:       Tuple		,		):


          '''simple docstring'''


          if num_proc is not None and num_proc <= 0:
              raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""					)

          UpperCAmelCase__     :						Any						=							dataset
          UpperCAmelCase__     :						int						=							name
          UpperCAmelCase__     :						Union[str, Any]						=							con
          UpperCAmelCase__     :						Any						=							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
          UpperCAmelCase__     :						int						=							num_proc
          UpperCAmelCase__     :						Optional[int]						=							to_sql_kwargs






      def lowercase_				(   self							:       int					):


          '''simple docstring'''


          UpperCAmelCase__     :						int						=							self.to_sql_kwargs.pop('''sql'''		,		_A					)
          UpperCAmelCase__     :						int						=							self.to_sql_kwargs.pop('''con'''		,		_A					)
          UpperCAmelCase__     :						List[Any]						=							self.to_sql_kwargs.pop('''index'''		,		_A					)

          UpperCAmelCase__     :						Optional[int]						=							self._write(index=_A		,		**self.to_sql_kwargs					)
          return written






      def lowercase_				(   self							:       Dict		,		_A							:       str					):


          '''simple docstring'''


          UpperCAmelCase__		,		UpperCAmelCase__		,		UpperCAmelCase__     :						int						=							args
          UpperCAmelCase__     :						int						=							{**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
          UpperCAmelCase__     :						Union[str, Any]						=							query_table(
              table=self.dataset.data		,		key=slice(_A		,		offset + self.batch_size					)		,		indices=self.dataset._indices		,		)
          UpperCAmelCase__     :						Tuple						=							batch.to_pandas()
          UpperCAmelCase__     :						Tuple						=							df.to_sql(self.name		,		self.con		,		index=_A		,		**_A					)
          return num_rows or len(_A					)






      def lowercase_				(   self							:       Optional[Any]		,		_A							:       Optional[int]		,		**_A							:       Tuple					):


          '''simple docstring'''


          UpperCAmelCase__     :						Any						=							0

          if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0		,		len(self.dataset					)		,		self.batch_size					)		,		unit='''ba'''		,		disable=not logging.is_progress_bar_enabled()		,		desc='''Creating SQL from Arrow format'''		,		):
                  written += self._batch_sql((offset, index, to_sql_kwargs)					)
          else:
              UpperCAmelCase__		,		UpperCAmelCase__     :						Optional[int]						=							len(self.dataset					), self.batch_size
              with multiprocessing.Pool(self.num_proc					) as pool:
                  for num_rows in logging.tqdm(
                      pool.imap(
                          self._batch_sql		,		[(offset, index, to_sql_kwargs) for offset in range(0		,		_A		,		_A					)]		,		)		,		total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,		unit='''ba'''		,		disable=not logging.is_progress_bar_enabled()		,		desc='''Creating SQL from Arrow format'''		,		):
                      written += num_rows

          return written







'''simple docstring'''




import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
class 			_lowerCAmelCase		(			A__	):






     """simple docstring"""






     snake_case_						      =   ["image_processor", "tokenizer"]
     snake_case_						      =   "LayoutLMv2ImageProcessor"
     snake_case_						      =   ("LayoutXLMTokenizer", "LayoutXLMTokenizerFast")



     def __init__(   self							:			List[Any]   ,       __snake_case							:			Union[str, Any]=None   ,       __snake_case							:			Optional[int]=None   ,       **__snake_case							:			Union[str, Any]						)->	str:
        if "feature_extractor" in kwargs:
           warnings.warn(
               """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
               """ instead."""   ,       __snake_case   ,       )
           snake_case 						=   kwargs.pop("""feature_extractor"""						)

        snake_case 						=   image_processor if image_processor is not None else feature_extractor
        if image_processor is None:
           raise ValueError("""You need to specify an `image_processor`."""						)
        if tokenizer is None:
           raise ValueError("""You need to specify a `tokenizer`."""						)

        super().__init__(__snake_case   ,       __snake_case						)



     def __call__(   self							:			Optional[Any]   ,       __snake_case							:			Dict   ,       __snake_case							:			Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None   ,       __snake_case							:			Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None   ,       __snake_case							:			Union[List[List[int]], List[List[List[int]]]] = None   ,       __snake_case							:			Optional[Union[List[int], List[List[int]]]] = None   ,       __snake_case							:			bool = True   ,       __snake_case							:			Union[bool, str, PaddingStrategy] = False   ,       __snake_case							:			Union[bool, str, TruncationStrategy] = None   ,       __snake_case							:			Optional[int] = None   ,       __snake_case							:			int = 0   ,       __snake_case							:			Optional[int] = None   ,       __snake_case							:			Optional[bool] = None   ,       __snake_case							:			Optional[bool] = None   ,       __snake_case							:			bool = False   ,       __snake_case							:			bool = False   ,       __snake_case							:			bool = False   ,       __snake_case							:			bool = False   ,       __snake_case							:			bool = True   ,       __snake_case							:			Optional[Union[str, TensorType]] = None   ,       **__snake_case							:			List[str]   ,       )->	BatchEncoding:
        # verify input
        if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               """You cannot provide bounding boxes """
               """if you initialized the image processor with apply_ocr set to True."""						)

        if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               """You cannot provide word labels if you initialized the image processor with apply_ocr set to True."""						)

        if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("""You cannot return overflowing tokens without returning the offsets mapping."""						)

        # first, apply the image processor
        snake_case 						=   self.image_processor(images=__snake_case   ,       return_tensors=__snake_case						)

        # second, apply the tokenizer
        if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(__snake_case   ,       __snake_case						):
              snake_case 						=   [text]  # add batch dimension (as the image processor always adds a batch dimension)
           snake_case 						=   features["""words"""]

        snake_case 						=   self.tokenizer(
            text=text if text is not None else features["""words"""]   ,       text_pair=text_pair if text_pair is not None else None   ,       boxes=boxes if boxes is not None else features["""boxes"""]   ,       word_labels=__snake_case   ,       add_special_tokens=__snake_case   ,       padding=__snake_case   ,       truncation=__snake_case   ,       max_length=__snake_case   ,       stride=__snake_case   ,       pad_to_multiple_of=__snake_case   ,       return_token_type_ids=__snake_case   ,       return_attention_mask=__snake_case   ,       return_overflowing_tokens=__snake_case   ,       return_special_tokens_mask=__snake_case   ,       return_offsets_mapping=__snake_case   ,       return_length=__snake_case   ,       verbose=__snake_case   ,       return_tensors=__snake_case   ,       **__snake_case   ,       )

        # add pixel values
        snake_case 						=   features.pop("""pixel_values"""						)
        if return_overflowing_tokens is True:
           snake_case 						=   self.get_overflowing_images(__snake_case   ,       encoded_inputs["""overflow_to_sample_mapping"""]						)
        snake_case 						=   images

        return encoded_inputs



     def      lowerCAmelCase			(   self							:			Any   ,       __snake_case							:			List[Any]   ,       __snake_case							:			str						)->	List[str]:
        # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
        snake_case 						=   []
        for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]						)

        if len(__snake_case						) != len(__snake_case						):
           raise ValueError(
               """Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"""
               f''' {len(__snake_case						)} and {len(__snake_case						)}'''						)

        return images_with_overflow



     def      lowerCAmelCase			(   self							:			int   ,       *__snake_case							:			Optional[int]   ,       **__snake_case							:			Tuple						)->	str:
        return self.tokenizer.batch_decode(*__snake_case   ,       **__snake_case						)



     def      lowerCAmelCase			(   self							:			Dict   ,       *__snake_case							:			Tuple   ,       **__snake_case							:			Optional[int]						)->	List[Any]:
        return self.tokenizer.decode(*__snake_case   ,       **__snake_case						)



     @property
     def      lowerCAmelCase			(   self							:			str						)->	int:
        return ["input_ids", "bbox", "attention_mask", "image"]



     @property
     def      lowerCAmelCase			(   self							:			Union[str, Any]						)->	List[str]:
        warnings.warn(
            """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""   ,       __snake_case   ,       )
        return self.image_processor_class



     @property
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Tuple:
        warnings.warn(
            """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""   ,       __snake_case   ,       )
        return self.image_processor

