






import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
			import sqlitea

			import sqlalchemy




class 							snake_case_       (    _SCREAMING_SNAKE_CASE      ):

						"""simple docstring"""



						def __init__(self:					str ,					__UpperCAmelCase:					str ,					__UpperCAmelCase:					Optional[int] ,					__UpperCAmelCase:					Tuple = None ,					__UpperCAmelCase:					Any = None ,					__UpperCAmelCase:					List[str] = False ,					**__UpperCAmelCase:					Tuple ,					)    ->     List[str]:




											'''simple docstring'''



											super().__init__(features=__UpperCAmelCase ,					cache_dir=__UpperCAmelCase ,					keep_in_memory=__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						List[str]      =					Sql(
											    cache_dir=__UpperCAmelCase ,					features=__UpperCAmelCase ,					sql=__UpperCAmelCase ,					con=__UpperCAmelCase ,					**__UpperCAmelCase ,					)



						def        UpperCAmelCase__					(self:					int   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Dict      =					None
											__a					:						Any      =					None
											__a					:						Any      =					None
											__a					:						Dict      =					None

											self.builder.download_and_prepare(
											    download_config=__UpperCAmelCase ,					download_mode=__UpperCAmelCase ,					verification_mode=__UpperCAmelCase ,					base_path=__UpperCAmelCase ,					)

											# Build dataset for splits
											__a					:						Optional[Any]      =					self.builder.as_dataset(
											    split="train" ,					verification_mode=__UpperCAmelCase ,					in_memory=self.keep_in_memory   )
											return dataset




class 							snake_case_       :

						"""simple docstring"""



						def __init__(self:					Tuple ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					Any = None ,					__UpperCAmelCase:					str = None ,					**__UpperCAmelCase:					Tuple ,					)    ->     int:




											'''simple docstring'''



											if num_proc is not None and num_proc <= 0:
																raise ValueError(f'num_proc {num_proc} must be an integer > 0.'   )

											__a					:						Any      =					dataset
											__a					:						str      =					name
											__a					:						Dict      =					con
											__a					:						str      =					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											__a					:						Any      =					num_proc
											__a					:						Optional[int]      =					to_sql_kwargs



						def        UpperCAmelCase__					(self:					Any   )    ->     int:




											'''simple docstring'''



											__a					:						Optional[int]      =					self.to_sql_kwargs.pop("sql" ,					__UpperCAmelCase   )
											__a					:						List[str]      =					self.to_sql_kwargs.pop("con" ,					__UpperCAmelCase   )
											__a					:						Optional[int]      =					self.to_sql_kwargs.pop("index" ,					__UpperCAmelCase   )

											__a					:						int      =					self._write(index=__UpperCAmelCase ,					**self.to_sql_kwargs   )
											return written



						def        UpperCAmelCase__					(self:					Union[str, Any] ,					__UpperCAmelCase:					Optional[Any]   )    ->     int:




											'''simple docstring'''



											__a					:						Optional[int]      =					args
											__a					:						Tuple      =					{**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
											__a					:						Union[str, Any]      =					query_table(
											    table=self.dataset.data ,					key=slice(__UpperCAmelCase ,					offset + self.batch_size   ) ,					indices=self.dataset._indices ,					)
											__a					:						Optional[int]      =					batch.to_pandas()
											__a					:						Tuple      =					df.to_sql(self.name ,					self.con ,					index=__UpperCAmelCase ,					**__UpperCAmelCase   )
											return num_rows or len(__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					Optional[Any] ,					**__UpperCAmelCase:					List[str]   )    ->     int:




											'''simple docstring'''



											__a					:						List[str]      =					0

											if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0 ,					len(self.dataset   ) ,					self.batch_size   ) ,					unit="ba" ,					disable=not logging.is_progress_bar_enabled() ,					desc="Creating SQL from Arrow format" ,					):
																					written += self._batch_sql((offset, index, to_sql_kwargs)   )
											else:
																__a					:						str      =					len(self.dataset   ), self.batch_size
																with multiprocessing.Pool(self.num_proc   ) as pool:
																					for num_rows in logging.tqdm(
																					    pool.imap(
																					        self._batch_sql ,					[(offset, index, to_sql_kwargs) for offset in range(0 ,					__UpperCAmelCase ,					__UpperCAmelCase   )] ,					) ,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size ,					unit="ba" ,					disable=not logging.is_progress_bar_enabled() ,					desc="Creating SQL from Arrow format" ,					):
																										written += num_rows

											return written








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase			:				Optional[int]   =     logging.get_logger(__name__)

lowercase			:				Optional[Any]   =     {"vocab_file": "vocab.txt"}

lowercase			:				Optional[int]   =     {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowercase			:				Tuple   =     {
    "YituTech/conv-bert-base": 5_12,
    "YituTech/conv-bert-medium-small": 5_12,
    "YituTech/conv-bert-small": 5_12,
}


lowercase			:				Optional[int]   =     {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class 					__lowercase  (						_SCREAMING_SNAKE_CASE				):





							"""simple docstring"""

							UpperCAmelCase_						:		int								=	VOCAB_FILES_NAMES
							UpperCAmelCase_						:		Optional[int]								=	PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_						:		List[Any]								=	PRETRAINED_INIT_CONFIGURATION
							UpperCAmelCase_						:		Optional[Any]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_						:		Optional[int]								=	ConvBertTokenizer




							def __init__(					self    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase="[UNK]"    ,						__UpperCAmelCase="[SEP]"    ,						__UpperCAmelCase="[PAD]"    ,						__UpperCAmelCase="[CLS]"    ,						__UpperCAmelCase="[MASK]"    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase=None    ,						**__UpperCAmelCase    ,						)		->       Optional[int]:
														super().__init__(
														    __UpperCAmelCase    ,						tokenizer_file=__UpperCAmelCase    ,						do_lower_case=__UpperCAmelCase    ,						unk_token=__UpperCAmelCase    ,						sep_token=__UpperCAmelCase    ,						pad_token=__UpperCAmelCase    ,						cls_token=__UpperCAmelCase    ,						mask_token=__UpperCAmelCase    ,						tokenize_chinese_chars=__UpperCAmelCase    ,						strip_accents=__UpperCAmelCase    ,						**__UpperCAmelCase    ,						)

														A      :					Any      		=					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
														if (
														    normalizer_state.get('''lowercase'''    ,						__UpperCAmelCase						) != do_lower_case
														    or normalizer_state.get('''strip_accents'''    ,						__UpperCAmelCase						) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''    ,						__UpperCAmelCase						) != tokenize_chinese_chars
														):
																					A      :					Tuple      		=					getattr(__UpperCAmelCase    ,						normalizer_state.pop('''type'''						)						)
																					A      :					int      		=					do_lower_case
																					A      :					int      		=					strip_accents
																					A      :					Union[str, Any]      		=					tokenize_chinese_chars
																					A      :					str      		=					normalizer_class(**__UpperCAmelCase						)

														A      :					str      		=					do_lower_case




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None						)		->       Optional[int]:
														A      :					Dict      		=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       List[int]:
														A      :					List[Any]      		=					[self.sep_token_id]
														A      :					Optional[Any]      		=					[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       Tuple[str]:
														A      :					Optional[int]      		=					self._tokenizer.model.save(__UpperCAmelCase    ,						name=__UpperCAmelCase						)
														return tuple(__UpperCAmelCase						)






