
'''simple docstring'''


import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__lowercase							:   Tuple					=     datasets.utils.logging.get_logger(__name__)

@dataclass
class 	__lowercase				(  datasets.BuilderConfig    ):
			lowerCamelCase			:		int	      =			1_0000
			lowerCamelCase			:		Optional[List[str]]	      =			None
			lowerCamelCase			:		Optional[datasets.Features]	      =			None

class 	__lowercase				(  datasets.ArrowBasedBuilder    ):
			lowerCamelCase			:		Tuple	      =			ParquetConfig

			def        UpperCAmelCase__ (self      ):
								return datasets.DatasetInfo(features=self.config.features      )

			def        UpperCAmelCase__ (self			,   A      ):
								if not self.config.data_files:
													raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""      )
								lowerCamelCase_					:      Union[str, Any]     =		dl_manager.download_and_extract(self.config.data_files      )
								if isinstance(A			,   (str, list, tuple)      ):
													lowerCamelCase_					:      Tuple     =		data_files
													if isinstance(A			,   A      ):
																		lowerCamelCase_					:      str     =		[files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													lowerCamelCase_					:      Dict     =		[dl_manager.iter_files(A      ) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,   gen_kwargs={'''files''': files}      )]
								lowerCamelCase_					:      List[str]     =		[]
								for split_name, files in data_files.items():
													if isinstance(A			,   A      ):
																		lowerCamelCase_					:      str     =		[files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													lowerCamelCase_					:      Optional[int]     =		[dl_manager.iter_files(A      ) for file in files]
													# Infer features is they are stoed in the arrow schema
													if self.info.features is None:
																		for file in itertools.chain.from_iterable(A      ):
																							with open(A			,   '''rb'''      ) as f:
																												lowerCamelCase_					:      Optional[int]     =		datasets.Features.from_arrow_schema(pq.read_schema(A      )      )
																							break
													splits.append(datasets.SplitGenerator(name=A			,   gen_kwargs={'''files''': files}      )      )
								return splits

			def        UpperCAmelCase__ (self			,   A      ):
								if self.info.features is not None:
													# more expensive cast to support nested features with keys in a different order
													# allows str <-> int/float or str to Audio for example
													lowerCamelCase_					:      int     =		table_cast(A			,   self.info.features.arrow_schema      )
								return pa_table






			def        UpperCAmelCase__ (self			,   A      ):
								lowerCamelCase_					:      Tuple     =		self.info.features.arrow_schema if self.info.features is not None else None
								if self.info.features is not None and self.config.columns is not None:
													if sorted(field.name for field in schema      ) != sorted(self.config.columns      ):
																		raise ValueError(
																		    F"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"""      )
								for file_idx, file in enumerate(itertools.chain.from_iterable(A      )      ):
													with open(A			,   '''rb'''      ) as f:
																		lowerCamelCase_					:      int     =		pq.ParquetFile(A      )
																		try:
																							for batch_idx, record_batch in enumerate(
																							    parquet_file.iter_batches(batch_size=self.config.batch_size			,   columns=self.config.columns      )      ):
																												lowerCamelCase_					:      Optional[int]     =		pa.Table.from_batches([record_batch]      )
																												# Uncomment for debugging (will print the Arrow table size and elements)
																												# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																												# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																												yield F"""{file_idx}_{batch_idx}""", self._cast_table(A      )
																		except ValueError as e:
																							logger.error(F"""Failed to read file '{file}' with error {type(A      )}: {e}"""      )
																							raise




'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__lowercase							:   int					=     {}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:   Any					=     ['''BartphoTokenizer''']

if TYPE_CHECKING:
					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_bartpho import BartphoTokenizer

else:
					import sys

					__lowercase							:   Union[str, Any]					=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


