






'''simple docstring'''







# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


__lowerCAmelCase									=   get_logger(__name__)





class 						SCREAMING_SNAKE_CASE       :
			snake_case__	=					"dummy_data"
			snake_case__	=					"datasets"
			snake_case__	=					False






			def __init__(			self   :      Optional[int]	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      Union[Version, str]	,    __SCREAMING_SNAKE_CASE   :      Optional[str] = None	,    __SCREAMING_SNAKE_CASE   :      bool = False	,    __SCREAMING_SNAKE_CASE   :      bool = True	,    __SCREAMING_SNAKE_CASE   :      Optional[List[Callable]] = None	,    )      ->      List[str]:
							a_		:	Any									=   0
							a_		:	Tuple									=   dataset_name
							a_		:	Dict									=   cache_dir
							a_		:	Optional[Any]									=   use_local_dummy_data
							a_		:	int									=   config
							# download_callbacks take a single url as input
							a_		:	List[Callable]									=   download_callbacks or []
							# if False, it doesn't load existing files and it returns the paths of the dummy files relative
							# to the dummy_data zip file root
							a_		:	List[Any]									=   load_existing_dummy_data

							# TODO(PVP, QL) might need to make this more general
							a_		:	str									=   str(__SCREAMING_SNAKE_CASE       )
							# to be downloaded
							a_		:	Optional[Any]									=   None
							a_		:	Union[str, Any]									=   None






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Tuple:
							if self._dummy_file is None:
											a_		:	Dict									=   self.download_dummy_data()
							return self._dummy_file






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]       )      ->      str:
							if self.config is not None:
											# structure is dummy / config_name / version_name
											return os.path.join('''dummy'''	,    self.config.name	,    self.version_name       )
							# structure is dummy / version_name
							return os.path.join('''dummy'''	,    self.version_name       )






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]       )      ->      List[Any]:
							return os.path.join(self.dummy_data_folder	,    '''dummy_data.zip'''       )






			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      List[str]:
							a_		:	List[Any]									=   (
							    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
							)

							a_		:	Any									=   cached_path(
							    __SCREAMING_SNAKE_CASE	,    cache_dir=self.cache_dir	,    extract_compressed_file=__SCREAMING_SNAKE_CASE	,    force_extract=__SCREAMING_SNAKE_CASE       )

							return os.path.join(__SCREAMING_SNAKE_CASE	,    self.dummy_file_name       )






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      Any       )      ->      Optional[int]:
							return os.path.join(self.datasets_scripts_dir	,    self.dataset_name	,    self.dummy_zip_file       )






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Dict:
							if self._bucket_url is None:
											a_		:	str									=   hf_github_url(self.dataset_name	,    self.dummy_zip_file.replace(os.sep	,    '''/'''       )       )
							return self._bucket_url






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      List[Any]       )      ->      Optional[int]:
							# return full path if its a dir
							if os.path.isdir(self.dummy_file       ):
											return self.dummy_file
							# else cut off path to file -> example `xsum`.
							return "/".join(self.dummy_file.replace(os.sep	,    '''/'''       ).split('''/'''       )[:-1]       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Dict	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    *__SCREAMING_SNAKE_CASE   :      Union[str, Any]       )      ->      Optional[Any]:
							if self.load_existing_dummy_data:
											# dummy data is downloaded and tested
											a_		:	int									=   self.dummy_file
							else:
											# dummy data cannot be downloaded and only the path to dummy file is returned
											a_		:	Dict									=   self.dummy_file_name

							# special case when data_url is a dict
							if isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
											return self.create_dummy_data_dict(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
							elif isinstance(__SCREAMING_SNAKE_CASE	,    (list, tuple)       ):
											return self.create_dummy_data_list(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
							else:
											return self.create_dummy_data_single(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    *__SCREAMING_SNAKE_CASE   :      int       )      ->      Any:
							return self.download_and_extract(__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Dict	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      str       )      ->      Any:
							return self.download_and_extract(__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Any	,    __SCREAMING_SNAKE_CASE   :      Optional[int]	,    *__SCREAMING_SNAKE_CASE   :      str	,    **__SCREAMING_SNAKE_CASE   :      Dict       )      ->      Optional[Any]:
							return path






			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      Any:
							return {}






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Union[str, Any]       )      ->      List[Any]:
							a_		:	List[str]									=   {}
							for key, single_urls in data_url.items():
											for download_callback in self.download_callbacks:
															if isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
																			for single_url in single_urls:
																							download_callback(__SCREAMING_SNAKE_CASE       )
															else:
																			a_		:	Optional[Any]									=   single_urls
																			download_callback(__SCREAMING_SNAKE_CASE       )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
											if isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
															a_		:	List[str]									=   [os.path.join(__SCREAMING_SNAKE_CASE	,    urllib.parse.quote_plus(Path(__SCREAMING_SNAKE_CASE       ).name       )       ) for x in single_urls]
											else:
															a_		:	Tuple									=   single_urls
															a_		:	str									=   os.path.join(__SCREAMING_SNAKE_CASE	,    urllib.parse.quote_plus(Path(__SCREAMING_SNAKE_CASE       ).name       )       )
											a_		:	List[str]									=   value

							# make sure that values are unique
							if all(isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ) for i in dummy_data_dict.values()       ) and len(set(dummy_data_dict.values()       )       ) < len(
							    dummy_data_dict.values()       ):
											# append key to value to make its name unique
											a_		:	Optional[Any]									=   {key: value + key for key, value in dummy_data_dict.items()}

							return dummy_data_dict






			def       SCREAMING_SNAKE_CASE   (			self   :      Any	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Dict       )      ->      Dict:
							a_		:	List[Any]									=   []
							# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
							a_		:	Any									=   all(bool(re.findall('''[0-9]{3,}-of-[0-9]{3,}'''	,    __SCREAMING_SNAKE_CASE       )       ) for url in data_url       )
							a_		:	List[str]									=   all(
							    url.startswith('''https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed'''       ) for url in data_url       )
							if data_url and (is_tf_records or is_pubmed_records):
											a_		:	Union[str, Any]									=   [data_url[0]] * len(__SCREAMING_SNAKE_CASE       )
							for single_url in data_url:
											for download_callback in self.download_callbacks:
															download_callback(__SCREAMING_SNAKE_CASE       )
											# we force the name of each key to be the last file / folder name of the url path
											# if the url has arguments, we need to encode them with urllib.parse.quote_plus
											a_		:	Any									=   os.path.join(__SCREAMING_SNAKE_CASE	,    urllib.parse.quote_plus(single_url.split('''/'''       )[-1]       )       )
											dummy_data_list.append(__SCREAMING_SNAKE_CASE       )
							return dummy_data_list






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      List[Any]       )      ->      Tuple:
							for download_callback in self.download_callbacks:
											download_callback(__SCREAMING_SNAKE_CASE       )
							# we force the name of each key to be the last file / folder name of the url path
							# if the url has arguments, we need to encode them with urllib.parse.quote_plus
							a_		:	Optional[int]									=   os.path.join(__SCREAMING_SNAKE_CASE	,    urllib.parse.quote_plus(data_url.split('''/'''       )[-1]       )       )
							if os.path.exists(__SCREAMING_SNAKE_CASE       ) or not self.load_existing_dummy_data:
											return value
							else:
											# Backward compatibility, maybe deprecate at one point.
											# For many datasets with single url calls to dl_manager.download_and_extract,
											# the dummy_data.zip file is actually the zipped downloaded file
											# while now we expected the dummy_data.zip file to be a directory containing
											# the downloaded file.
											return path_to_dummy_data






			def       SCREAMING_SNAKE_CASE   (			self   :      List[Any]       )      ->      str:
							pass






			def       SCREAMING_SNAKE_CASE   (			self   :      Union[str, Any]       )      ->      List[Any]:
							pass






			def       SCREAMING_SNAKE_CASE   (			self   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]       )      ->      Tuple:
							def _iter_archive_members(__SCREAMING_SNAKE_CASE   :      Optional[Any]       ):
											# this preserves the order of the members inside the ZIP archive
											a_		:	Any									=   Path(self.dummy_file       ).parent
											a_		:	int									=   path.relative_to(__SCREAMING_SNAKE_CASE       )
											with ZipFile(self.local_path_to_dummy_data       ) as zip_file:
															a_		:	Tuple									=   zip_file.namelist()
											for member in members:
															if member.startswith(relative_path.as_posix()       ):
																			yield dummy_parent_path.joinpath(__SCREAMING_SNAKE_CASE       )

							a_		:	Dict									=   Path(__SCREAMING_SNAKE_CASE       )
							a_		:	Union[str, Any]									=   _iter_archive_members(__SCREAMING_SNAKE_CASE       ) if self.use_local_dummy_data else path.rglob('''*'''       )
							for file_path in file_paths:
											if file_path.is_file() and not file_path.name.startswith(('''.''', '''__''')       ):
															yield file_path.relative_to(__SCREAMING_SNAKE_CASE       ).as_posix(), file_path.open('''rb'''       )







			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]       )      ->      Optional[Any]:
							if not isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
											a_		:	Dict									=   [paths]
							for path in paths:
											if os.path.isfile(__SCREAMING_SNAKE_CASE       ):
															if os.path.basename(__SCREAMING_SNAKE_CASE       ).startswith(('''.''', '''__''')       ):
																			return
															yield path
											else:
															for dirpath, dirnames, filenames in os.walk(__SCREAMING_SNAKE_CASE       ):
																			if os.path.basename(__SCREAMING_SNAKE_CASE       ).startswith(('''.''', '''__''')       ):
																							continue
																			dirnames.sort()
																			for filename in sorted(__SCREAMING_SNAKE_CASE       ):
																							if filename.startswith(('''.''', '''__''')       ):
																											continue
																							yield os.path.join(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )












'''simple docstring'''







import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


__lowerCAmelCase									=   datasets.utils.logging.get_logger(__name__)

__lowerCAmelCase									=   ['names', 'prefix']
__lowerCAmelCase									=   ['warn_bad_lines', 'error_bad_lines', 'mangle_dupe_cols']
__lowerCAmelCase									=   ['encoding_errors', 'on_bad_lines']
__lowerCAmelCase									=   ['date_format']





@dataclass
class 						SCREAMING_SNAKE_CASE       (       datasets.BuilderConfig							):
			snake_case__	=					","
			snake_case__	=					None
			snake_case__	=					"infer"
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					True
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					False
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					True
			snake_case__	=					True
			snake_case__	=					False
			snake_case__	=					True
			snake_case__	=					None
			snake_case__	=					"."
			snake_case__	=					None
			snake_case__	=					'"'
			snake_case__	=					0
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					None
			snake_case__	=					True
			snake_case__	=					True
			snake_case__	=					0
			snake_case__	=					True
			snake_case__	=					False
			snake_case__	=					None
			snake_case__	=					10000
			snake_case__	=					None
			snake_case__	=					"strict"
			snake_case__	=					"error"
			snake_case__	=					None






			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      Optional[int]:
							if self.delimiter is not None:
											a_		:	Optional[Any]									=   self.delimiter
							if self.column_names is not None:
											a_		:	Optional[Any]									=   self.column_names







			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      Optional[Any]:
							a_		:	List[Any]									=   {
							    '''sep''': self.sep,
							    '''header''': self.header,
							    '''names''': self.names,
							    '''index_col''': self.index_col,
							    '''usecols''': self.usecols,
							    '''prefix''': self.prefix,
							    '''mangle_dupe_cols''': self.mangle_dupe_cols,
							    '''engine''': self.engine,
							    '''converters''': self.converters,
							    '''true_values''': self.true_values,
							    '''false_values''': self.false_values,
							    '''skipinitialspace''': self.skipinitialspace,
							    '''skiprows''': self.skiprows,
							    '''nrows''': self.nrows,
							    '''na_values''': self.na_values,
							    '''keep_default_na''': self.keep_default_na,
							    '''na_filter''': self.na_filter,
							    '''verbose''': self.verbose,
							    '''skip_blank_lines''': self.skip_blank_lines,
							    '''thousands''': self.thousands,
							    '''decimal''': self.decimal,
							    '''lineterminator''': self.lineterminator,
							    '''quotechar''': self.quotechar,
							    '''quoting''': self.quoting,
							    '''escapechar''': self.escapechar,
							    '''comment''': self.comment,
							    '''encoding''': self.encoding,
							    '''dialect''': self.dialect,
							    '''error_bad_lines''': self.error_bad_lines,
							    '''warn_bad_lines''': self.warn_bad_lines,
							    '''skipfooter''': self.skipfooter,
							    '''doublequote''': self.doublequote,
							    '''memory_map''': self.memory_map,
							    '''float_precision''': self.float_precision,
							    '''chunksize''': self.chunksize,
							    '''encoding_errors''': self.encoding_errors,
							    '''on_bad_lines''': self.on_bad_lines,
							    '''date_format''': self.date_format,
							}

							# some kwargs must not be passed if they don't have a default value
							# some others are deprecated and we can also not pass them if they are the default value
							for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
											if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()	,    __SCREAMING_SNAKE_CASE       ):
															del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
							if not (datasets.config.PANDAS_VERSION.major >= 2):
											for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
															del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
							if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
											for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
															del pd_read_csv_kwargs[pd_read_csv_parameter]

							return pd_read_csv_kwargs





class 						SCREAMING_SNAKE_CASE       (       datasets.ArrowBasedBuilder							):
			snake_case__	=					CsvConfig






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      Tuple:
							return datasets.DatasetInfo(features=self.config.features       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Optional[int]       )      ->      Any:
							if not self.config.data_files:
											raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'       )
							a_		:	Tuple									=   dl_manager.download_and_extract(self.config.data_files       )
							if isinstance(__SCREAMING_SNAKE_CASE	,    (str, list, tuple)       ):
											a_		:	int									=   data_files
											if isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
															a_		:	Optional[Any]									=   [files]
											a_		:	Optional[Any]									=   [dl_manager.iter_files(__SCREAMING_SNAKE_CASE       ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,    gen_kwargs={'''files''': files}       )]
							a_		:	List[str]									=   []
							for split_name, files in data_files.items():
											if isinstance(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
															a_		:	int									=   [files]
											a_		:	Union[str, Any]									=   [dl_manager.iter_files(__SCREAMING_SNAKE_CASE       ) for file in files]
											splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE	,    gen_kwargs={'''files''': files}       )       )
							return splits






			def       SCREAMING_SNAKE_CASE   (			self   :      Union[str, Any]	,    __SCREAMING_SNAKE_CASE   :      pa.Table       )      ->      pa.Table:
							if self.config.features is not None:
											a_		:	Any									=   self.config.features.arrow_schema
											if all(not require_storage_cast(__SCREAMING_SNAKE_CASE       ) for feature in self.config.features.values()       ):
															# cheaper cast
															a_		:	List[Any]									=   pa.Table.from_arrays([pa_table[field.name] for field in schema]	,    schema=__SCREAMING_SNAKE_CASE       )
											else:
															# more expensive cast; allows str <-> int/float or str to Audio for example
															a_		:	Optional[Any]									=   table_cast(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
							return pa_table







			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[int]	,    __SCREAMING_SNAKE_CASE   :      Any       )      ->      Optional[int]:
							a_		:	Union[str, Any]									=   self.config.features.arrow_schema if self.config.features else None
							# dtype allows reading an int column as str
							a_		:	Optional[int]									=   (
							    {
							        name: dtype.to_pandas_dtype() if not require_storage_cast(__SCREAMING_SNAKE_CASE       ) else object
							        for name, dtype, feature in zip(schema.names	,    schema.types	,    self.config.features.values()       )
							    }
							    if schema is not None
							    else None
							)
							for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE       )       ):
											a_		:	Tuple									=   pd.read_csv(__SCREAMING_SNAKE_CASE	,    iterator=__SCREAMING_SNAKE_CASE	,    dtype=__SCREAMING_SNAKE_CASE	,    **self.config.pd_read_csv_kwargs       )
											try:
															for batch_idx, df in enumerate(__SCREAMING_SNAKE_CASE       ):
																			a_		:	Optional[int]									=   pa.Table.from_pandas(__SCREAMING_SNAKE_CASE       )
																			# Uncomment for debugging (will print the Arrow table size and elements)
																			# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																			# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																			yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE       )
											except ValueError as e:
															logger.error(f'Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE       )}: {e}'       )
															raise




